diff --git a/db/docs/batched_background_migrations/backfill_pipeline_execution_policies_config_links.yml b/db/docs/batched_background_migrations/backfill_pipeline_execution_policies_config_links.yml new file mode 100644 index 0000000000000000000000000000000000000000..ef896ac716cd2824cc08e59853e1fb535a558181 --- /dev/null +++ b/db/docs/batched_background_migrations/backfill_pipeline_execution_policies_config_links.yml @@ -0,0 +1,8 @@ +--- +migration_job_name: BackfillPipelineExecutionPoliciesConfigLinks +description: Backfill links between Pipeline execution policies and projects where their configuration is stored. +feature_category: security_policy_management +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/181151 +milestone: '17.10' +queued_migration_version: 20250205175341 +finalized_by: # version of the migration that finalized this BBM diff --git a/db/post_migrate/20250205175341_queue_backfill_pipeline_execution_policies_config_links.rb b/db/post_migrate/20250205175341_queue_backfill_pipeline_execution_policies_config_links.rb new file mode 100644 index 0000000000000000000000000000000000000000..ba70cf2ad3bcac9b7958c273369ab56487de9c98 --- /dev/null +++ b/db/post_migrate/20250205175341_queue_backfill_pipeline_execution_policies_config_links.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class QueueBackfillPipelineExecutionPoliciesConfigLinks < Gitlab::Database::Migration[2.2] + milestone '17.10' + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + MIGRATION = "BackfillPipelineExecutionPoliciesConfigLinks" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + def up + queue_batched_background_migration( + MIGRATION, + :security_policies, + :id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :security_policies, :id, []) + end +end diff --git a/db/schema_migrations/20250205175341 b/db/schema_migrations/20250205175341 new file mode 100644 index 0000000000000000000000000000000000000000..7ac0a533687a2a193c38cba586a0dc23c246c4d7 --- /dev/null +++ b/db/schema_migrations/20250205175341 @@ -0,0 +1 @@ +db6a63a66ce4d052db0ed5815fb3e9822de90ccb734b49bda9d71d070dcecd48 \ No newline at end of file diff --git a/ee/config/feature_flags/wip/unblock_rules_using_pipeline_execution_policies.yml b/ee/config/feature_flags/wip/unblock_rules_using_pipeline_execution_policies.yml index bcf0396720eac876e0d00bd168ba099628297397..21dd5a228dca4970299409694a1d0643bef9bf54 100644 --- a/ee/config/feature_flags/wip/unblock_rules_using_pipeline_execution_policies.yml +++ b/ee/config/feature_flags/wip/unblock_rules_using_pipeline_execution_policies.yml @@ -2,7 +2,7 @@ name: unblock_rules_using_pipeline_execution_policies feature_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/498624 introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/179741 -rollout_issue_url: +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/520982 milestone: '17.9' group: group::security policies type: wip diff --git a/ee/lib/ee/gitlab/background_migration/backfill_pipeline_execution_policies_config_links.rb b/ee/lib/ee/gitlab/background_migration/backfill_pipeline_execution_policies_config_links.rb new file mode 100644 index 0000000000000000000000000000000000000000..ebd249a7311c9ce81fc52ca485479e39fa9e258d --- /dev/null +++ b/ee/lib/ee/gitlab/background_migration/backfill_pipeline_execution_policies_config_links.rb @@ -0,0 +1,169 @@ +# frozen_string_literal: true + +module EE + module Gitlab + module BackgroundMigration + # rubocop:disable Layout/LineLength -- unavoidable long class names + module BackfillPipelineExecutionPoliciesConfigLinks + extend ActiveSupport::Concern + extend ::Gitlab::Utils::Override + + PIPELINE_EXECUTION_POLICY_TYPE = 2 + + prepended do + operation_name :backfill_pipeline_execution_policies_config_links + scope_to ->(relation) { relation.where(type: PIPELINE_EXECUTION_POLICY_TYPE) } + end + + module CaseSensitivity + extend ActiveSupport::Concern + + class_methods do + def iwhere(params) + criteria = self + + params.each do |column, value| + column = arel_table[column] unless column.is_a?(Arel::Attribute) + + criteria = criteria.where(value_equal(column, value)) + end + + criteria + end + + private + + def value_equal(column, value) + lower_value = lower_value(value) + + lower_column(column).eq(lower_value).to_sql + end + + def lower_value(value) + Arel::Nodes::NamedFunction.new('LOWER', [Arel::Nodes.build_quoted(value)]) + end + + def lower_column(column) + column.lower + end + end + end + + class Route < ::ApplicationRecord + include CaseSensitivity + + self.table_name = 'routes' + end + + module Routable + extend ActiveSupport::Concern + include CaseSensitivity + + included do + has_one :route, as: :source + end + + # removed `follow_redirects` as it is not used in the application code + def self.find_by_full_path(path, route_scope: nil) + return unless path.present? + + path = path.to_s + + path_condition = { path: path } + + source_type_condition = { source_type: 'Project' } # changed manually for this migration to not use the scoped Project class name + + route = Route.where(source_type_condition).find_by(path_condition) || + Route.where(source_type_condition).iwhere(path_condition).take + + return unless route + return route.source unless route_scope + + route_scope.find_by(id: route.source_id) + end + + class_methods do + # removed `follow_redirects` as it is not used in the application code + def find_by_full_path(path) + route_scope = all + + Routable.find_by_full_path( + path, + route_scope: route_scope + ) + end + end + end + + class Project < ::ApplicationRecord + include Routable + + self.table_name = 'projects' + + belongs_to :parent, + class_name: '::EE::Gitlab::BackgroundMigration::BackfillPipelineExecutionPoliciesConfigLinks::Namespace' + has_one :route, as: :source, + class_name: '::EE::Gitlab::BackgroundMigration::BackfillPipelineExecutionPoliciesConfigLinks::Route' + belongs_to :namespace + end + + class Namespace < ::ApplicationRecord + include Routable + + self.table_name = 'namespaces' + self.inheritance_column = :_type_disabled + + belongs_to :parent, + class_name: '::EE::Gitlab::BackgroundMigration::BackfillPipelineExecutionPoliciesConfigLinks::Namespace' + end + + class PipelineExecutionPolicyConfigLink < ::ApplicationRecord + self.table_name = 'security_pipeline_execution_policy_config_links' + + belongs_to :security_policy, + class_name: '::EE::Gitlab::BackgroundMigration::BackfillPipelineExecutionPoliciesConfigLinks::SecurityPolicy', + inverse_of: :security_pipeline_execution_policy_config_link + belongs_to :project, class_name: '::EE::Gitlab::BackgroundMigration::BackfillPipelineExecutionPoliciesConfigLinks::Project' + end + + class SecurityPolicy < ::ApplicationRecord + self.table_name = 'security_policies' + self.inheritance_column = :_type_disabled + + enum type: { + approval_policy: 0, + scan_execution_policy: 1, + pipeline_execution_policy: 2, + vulnerability_management_policy: 3 + }, _prefix: true + + has_one :security_pipeline_execution_policy_config_link, + class_name: '::EE::Gitlab::BackgroundMigration::BackfillPipelineExecutionPoliciesConfigLinks::PipelineExecutionPolicyConfigLink', + inverse_of: :security_policy + + def pipeline_execution_ci_config + content&.dig('content', 'include', 0) + end + + def update_pipeline_execution_policy_config_link! + return unless type_pipeline_execution_policy? + + # Changed from the application code to avoid recreating existing links + return if security_pipeline_execution_policy_config_link.present? + + config_project = Project.find_by_full_path(pipeline_execution_ci_config['project']) + create_security_pipeline_execution_policy_config_link!(project: config_project) if config_project + end + end + + override :perform + def perform + each_sub_batch do |sub_batch| + SecurityPolicy.id_in(sub_batch).find_each(&:update_pipeline_execution_policy_config_link!) + end + end + end + # rubocop:enable Layout/LineLength + end + end +end diff --git a/ee/spec/lib/ee/gitlab/background_migration/backfill_pipeline_execution_policies_config_links_spec.rb b/ee/spec/lib/ee/gitlab/background_migration/backfill_pipeline_execution_policies_config_links_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..269a623aa1b9d77f2233b774b79072ca914c0c5d --- /dev/null +++ b/ee/spec/lib/ee/gitlab/background_migration/backfill_pipeline_execution_policies_config_links_spec.rb @@ -0,0 +1,162 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillPipelineExecutionPoliciesConfigLinks, feature_category: :security_policy_management do + let(:security_orchestration_policy_configurations) { table(:security_orchestration_policy_configurations) } + let(:security_policies) { table(:security_policies) } + let(:security_pipeline_execution_policy_config_links) { table(:security_pipeline_execution_policy_config_links) } + let(:organizations) { table(:organizations) } + let(:projects) { table(:projects) } + let(:routes) { table(:routes) } + let(:namespaces) { table(:namespaces) } + let(:organization) { organizations.create!(name: 'organization', path: 'organization') } + let(:namespace) { namespaces.create!(name: 'Test', path: 'test', organization_id: organization.id) } + let(:project_namespace) do + namespaces.create!(name: 'Project1', path: 'project_1', type: 'Project', organization_id: organization.id) + end + + let(:spp_project_namespace) do + namespaces.create!(name: 'Project2', path: 'spp', type: 'Project', organization_id: organization.id) + end + + let(:config_project_namespace) do + namespaces.create!(name: 'PEP config', path: 'pep-config', type: 'Project', organization_id: organization.id) + end + + let(:project) do + projects.create!( + name: 'project_1', + path: 'project_1', + namespace_id: namespace.id, + project_namespace_id: project_namespace.id, + organization_id: organization.id + ) + end + + let(:policy_project) do + projects.create!( + name: 'SPP', + path: 'spp', + namespace_id: namespace.id, + project_namespace_id: spp_project_namespace.id, + organization_id: organization.id + ) + end + + let!(:config_project) do + projects.create!( + name: 'PEP config', + path: 'pep-config', + namespace_id: namespace.id, + project_namespace_id: config_project_namespace.id, + organization_id: organization.id + ) + end + + let!(:config_project_route) do + routes.create!(path: 'pep-config', source_id: config_project.id, namespace_id: namespace.id, + source_type: 'Project') + end + + let(:configuration) do + security_orchestration_policy_configurations.create!( + security_policy_management_project_id: policy_project.id, + project_id: project.id + ) + end + + let(:pipeline_execution_policy_content) do + { include: [{ project: 'pep-config', file: 'compliance-pipeline.yml' }] } + end + + let!(:policy) do + security_policies.create!( + name: 'PEP', + security_orchestration_policy_configuration_id: configuration.id, + policy_index: 0, + checksum: '0' * 64, + security_policy_management_project_id: policy_project.id, + type: described_class::SecurityPolicy.types[:pipeline_execution_policy], + content: { + content: pipeline_execution_policy_content, + pipeline_config_strategy: 'inject_ci' + } + ) + end + + let(:args) do + min, max = security_policies.pick('MIN(id)', 'MAX(id)') + { + start_id: min, + end_id: max, + batch_table: 'security_policies', + batch_column: 'id', + sub_batch_size: 1000, + pause_ms: 0, + connection: ApplicationRecord.connection + } + end + + subject(:perform_migration) { described_class.new(**args).perform } + + shared_examples_for 'creates the link' do + it 'creates the link', :aggregate_failures do + expect { perform_migration }.to change { security_pipeline_execution_policy_config_links.count }.from(0).to(1) + + link = security_pipeline_execution_policy_config_links.first + expect(link.project_id).to eq(config_project.id) + expect(link.security_policy_id).to eq(policy.id) + end + end + + it_behaves_like 'creates the link' + + context 'when the links already exist' do + let!(:existing_link) do + security_pipeline_execution_policy_config_links.create!( + project_id: config_project.id, + security_policy_id: policy.id) + end + + it 'does not change the existing links' do + expect { perform_migration }.not_to change { existing_link.reload } + end + end + + context 'when PEP project is referenced as case-insensitive' do + let(:pipeline_execution_policy_content) do + { include: [{ project: 'PEP-CONFIG', file: 'compliance-pipeline.yml' }] } + end + + it_behaves_like 'creates the link' + end + + context 'when the referenced PEP project does not exist' do + let(:pipeline_execution_policy_content) do + { include: [{ project: 'pep-project-does-not-exist', file: 'compliance-pipeline.yml' }] } + end + + it 'does not create the link' do + expect { perform_migration }.not_to change { security_pipeline_execution_policy_config_links.count } + end + end + + context 'when policy is not a pipeline execution policy' do + let!(:policy) do + security_policies.create!( + name: 'Approval policy', + security_orchestration_policy_configuration_id: configuration.id, + policy_index: 0, + checksum: '0' * 64, + security_policy_management_project_id: policy_project.id, + type: described_class::SecurityPolicy.types[:approval_policy], + content: {} + ) + end + + it 'does not create the link' do + expect { perform_migration }.not_to change { security_pipeline_execution_policy_config_links.count } + end + end +end diff --git a/lib/gitlab/background_migration/backfill_pipeline_execution_policies_config_links.rb b/lib/gitlab/background_migration/backfill_pipeline_execution_policies_config_links.rb new file mode 100644 index 0000000000000000000000000000000000000000..bf55ef3da8ad849ace8e94fce4cf0d20eddfed8a --- /dev/null +++ b/lib/gitlab/background_migration/backfill_pipeline_execution_policies_config_links.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillPipelineExecutionPoliciesConfigLinks < BatchedMigrationJob + feature_category :security_policy_management + + def perform; end + end + end +end + +Gitlab::BackgroundMigration::BackfillPipelineExecutionPoliciesConfigLinks.prepend_mod diff --git a/spec/migrations/20250205175341_queue_backfill_pipeline_execution_policies_config_links_spec.rb b/spec/migrations/20250205175341_queue_backfill_pipeline_execution_policies_config_links_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..0febbd5a57c38099407d33b051abee9e21a17fbc --- /dev/null +++ b/spec/migrations/20250205175341_queue_backfill_pipeline_execution_policies_config_links_spec.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillPipelineExecutionPoliciesConfigLinks, migration: :gitlab_main_cell, feature_category: :security_policy_management do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + gitlab_schema: :gitlab_main, + table_name: :security_policies, + column_name: :id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE + ) + } + end + end +end