diff --git a/.rubocop_todo/lint/redundant_cop_disable_directive.yml b/.rubocop_todo/lint/redundant_cop_disable_directive.yml index 6653f702adb1b25abc905c55b43541fa892110ba..147f47325ddbbabecb397320f75c6b8ee5803d06 100644 --- a/.rubocop_todo/lint/redundant_cop_disable_directive.yml +++ b/.rubocop_todo/lint/redundant_cop_disable_directive.yml @@ -380,7 +380,6 @@ Lint/RedundantCopDisableDirective: - 'spec/lib/backup/manager_spec.rb' - 'spec/lib/gitlab/avatar_cache_spec.rb' - 'spec/lib/gitlab/background_migration/backfill_finding_id_in_vulnerabilities_spec.rb' - - 'spec/lib/gitlab/background_migration/backfill_has_merge_request_of_vulnerability_reads_spec.rb' - 'spec/lib/gitlab/background_migration/backfill_missing_ci_cd_settings_spec.rb' - 'spec/lib/gitlab/background_migration/backfill_root_storage_statistics_fork_storage_sizes_spec.rb' - 'spec/lib/gitlab/background_migration/backfill_uuid_conversion_column_in_vulnerability_occurrences_spec.rb' diff --git a/.rubocop_todo/style/inline_disable_annotation.yml b/.rubocop_todo/style/inline_disable_annotation.yml index f15d7897f1585fa20ca67764a35ce2255b4edb06..bda46cbd1146f899e235396de992f374238e958d 100644 --- a/.rubocop_todo/style/inline_disable_annotation.yml +++ b/.rubocop_todo/style/inline_disable_annotation.yml @@ -2547,7 +2547,6 @@ Style/InlineDisableAnnotation: - 'spec/lib/gitlab/authorized_keys_spec.rb' - 'spec/lib/gitlab/avatar_cache_spec.rb' - 'spec/lib/gitlab/background_migration/backfill_finding_id_in_vulnerabilities_spec.rb' - - 'spec/lib/gitlab/background_migration/backfill_has_merge_request_of_vulnerability_reads_spec.rb' - 'spec/lib/gitlab/background_migration/backfill_missing_ci_cd_settings_spec.rb' - 'spec/lib/gitlab/background_migration/backfill_project_repositories_spec.rb' - 'spec/lib/gitlab/background_migration/backfill_root_storage_statistics_fork_storage_sizes_spec.rb' diff --git a/db/docs/batched_background_migrations/backfill_has_merge_request_of_vulnerability_reads.yml b/db/docs/batched_background_migrations/backfill_has_merge_request_of_vulnerability_reads.yml deleted file mode 100644 index 8cce41600dd8a643bf9134b5ef13590f4e84a594..0000000000000000000000000000000000000000 --- a/db/docs/batched_background_migrations/backfill_has_merge_request_of_vulnerability_reads.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -migration_job_name: BackfillHasMergeRequestOfVulnerabilityReads -description: Backfills has_merge_request column for vulnerability_reads table. -feature_category: database -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/130952 -milestone: '16.4' -finalized_by: '20240422232001' diff --git a/db/post_migrate/20230907155247_queue_backfill_has_merge_request_of_vulnerability_reads.rb b/db/post_migrate/20230907155247_queue_backfill_has_merge_request_of_vulnerability_reads.rb deleted file mode 100644 index ed767502b0b7cc6933b3929bf43cd8fa2492990a..0000000000000000000000000000000000000000 --- a/db/post_migrate/20230907155247_queue_backfill_has_merge_request_of_vulnerability_reads.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -class QueueBackfillHasMergeRequestOfVulnerabilityReads < Gitlab::Database::Migration[2.1] - MIGRATION = 'BackfillHasMergeRequestOfVulnerabilityReads' - DELAY_INTERVAL = 2.minutes - BATCH_SIZE = 10_000 - SUB_BATCH_SIZE = 200 - - restrict_gitlab_migration gitlab_schema: :gitlab_main - disable_ddl_transaction! - - def up - Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.with_suppressed do - queue_batched_background_migration( - MIGRATION, - :vulnerability_reads, - :vulnerability_id, - job_interval: DELAY_INTERVAL, - batch_size: BATCH_SIZE, - sub_batch_size: SUB_BATCH_SIZE - ) - end - end - - def down - Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.with_suppressed do - delete_batched_background_migration(MIGRATION, :vulnerability_reads, :vulnerability_id, []) - end - end -end diff --git a/db/post_migrate/20240422232001_finalize_backfill_has_merge_request_of_vulnerability_reads.rb b/db/post_migrate/20240422232001_finalize_backfill_has_merge_request_of_vulnerability_reads.rb deleted file mode 100644 index c05dfa226504f1693412f239610ff3e70bdba6a3..0000000000000000000000000000000000000000 --- a/db/post_migrate/20240422232001_finalize_backfill_has_merge_request_of_vulnerability_reads.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -class FinalizeBackfillHasMergeRequestOfVulnerabilityReads < Gitlab::Database::Migration[2.2] - milestone '17.0' - - disable_ddl_transaction! - - restrict_gitlab_migration gitlab_schema: :gitlab_main - - def up - ensure_batched_background_migration_is_finished( - job_class_name: 'BackfillHasMergeRequestOfVulnerabilityReads', - table_name: :vulnerability_reads, - column_name: :vulnerability_id, - job_arguments: [], - finalize: true - ) - end - - def down; end -end diff --git a/db/post_migrate/20240724182445_update_finish_status_for_bbm_migrations.rb b/db/post_migrate/20240724182445_update_finish_status_for_bbm_migrations.rb index 90c04352be0bc2ab25be30fc6b3b24c22883f0a0..889c4ea3ade2cdcb2e543835073f7dcc027c6677 100644 --- a/db/post_migrate/20240724182445_update_finish_status_for_bbm_migrations.rb +++ b/db/post_migrate/20240724182445_update_finish_status_for_bbm_migrations.rb @@ -27,7 +27,6 @@ class UpdateFinishStatusForBbmMigrations < Gitlab::Database::Migration[2.2] BackfillFindingIdInVulnerabilities BackfillFindingInitialPipelineId BackfillHasIssuesForExternalIssueLinks - BackfillHasMergeRequestOfVulnerabilityReads BackfillHasRemediationsOfVulnerabilityReads BackfillIssueSearchDataNamespaceId BackfillMergeRequestDiffsProjectId diff --git a/db/schema_migrations/20230907155247 b/db/schema_migrations/20230907155247 deleted file mode 100644 index 6d709e8c35c6b4c5e8eddff1bccd3a6420a2d033..0000000000000000000000000000000000000000 --- a/db/schema_migrations/20230907155247 +++ /dev/null @@ -1 +0,0 @@ -969028a44aa3e656595c2af113fab7a82f8f28514337b97bfb467a5c5550dfc3 \ No newline at end of file diff --git a/db/schema_migrations/20240422232001 b/db/schema_migrations/20240422232001 deleted file mode 100644 index 4d1003536b401d48d3d9ac564d50ea4ebc71c212..0000000000000000000000000000000000000000 --- a/db/schema_migrations/20240422232001 +++ /dev/null @@ -1 +0,0 @@ -7a2458be07d27f56db09836189caf2abbbb495f1ad3e59e55f3d0e366ee14e98 \ No newline at end of file diff --git a/lib/gitlab/background_migration/backfill_has_merge_request_of_vulnerability_reads.rb b/lib/gitlab/background_migration/backfill_has_merge_request_of_vulnerability_reads.rb deleted file mode 100644 index 69f3fb3a1faa4214ff3f1c21c0b9ff37a0e15335..0000000000000000000000000000000000000000 --- a/lib/gitlab/background_migration/backfill_has_merge_request_of_vulnerability_reads.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Sets the `has_merge_request` of the existing `vulnerability_reads` records - class BackfillHasMergeRequestOfVulnerabilityReads < BatchedMigrationJob - operation_name :set_has_merge_request - feature_category :database - - UPDATE_SQL = <<~SQL - UPDATE - vulnerability_reads - SET - has_merge_request = true - FROM - (%<subquery>s) as sub_query - WHERE - vulnerability_reads.vulnerability_id = sub_query.vulnerability_id - SQL - - def perform - each_sub_batch do |sub_batch| - update_query = update_query_for(sub_batch) - - ::Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification.temporary_ignore_tables_in_transaction( - %w[vulnerability_reads vulnerability_merge_request_links], - url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/480545' - ) do - ::Gitlab::Database.allow_cross_joins_across_databases( - url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/480545' - ) do - connection.execute(update_query) - end - end - end - end - - private - - def update_query_for(sub_batch) - subquery = sub_batch.joins(" - INNER JOIN vulnerability_merge_request_links ON - vulnerability_reads.vulnerability_id = - vulnerability_merge_request_links.vulnerability_id") - - format(UPDATE_SQL, subquery: subquery.to_sql) - end - end - end -end diff --git a/spec/lib/gitlab/background_migration/backfill_has_merge_request_of_vulnerability_reads_spec.rb b/spec/lib/gitlab/background_migration/backfill_has_merge_request_of_vulnerability_reads_spec.rb deleted file mode 100644 index fc4597fbb9633f385f67298959588c044d47a8fc..0000000000000000000000000000000000000000 --- a/spec/lib/gitlab/background_migration/backfill_has_merge_request_of_vulnerability_reads_spec.rb +++ /dev/null @@ -1,101 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::BackfillHasMergeRequestOfVulnerabilityReads, schema: 20230907155247, feature_category: :database do # rubocop:disable Layout/LineLength - let(:namespaces) { table(:namespaces) } - let(:projects) { table(:projects) } - let(:users) { table(:users) } - let(:scanners) { table(:vulnerability_scanners) } - let(:vulnerabilities) { table(:vulnerabilities) } - let(:vulnerability_reads) { table(:vulnerability_reads) } - let(:merge_requests) { table(:merge_requests) } - let(:merge_request_links) { table(:vulnerability_merge_request_links) } - - let(:namespace) { namespaces.create!(name: 'user', path: 'user') } - let(:project) { projects.create!(namespace_id: namespace.id, project_namespace_id: namespace.id) } - let(:user) { users.create!(username: 'john_doe', email: 'johndoe@gitlab.com', projects_limit: 10) } - let(:scanner) { scanners.create!(project_id: project.id, external_id: 'external_id', name: 'Test Scanner') } - - let(:vulnerability) do - vulnerabilities.create!( - project_id: project.id, - author_id: user.id, - title: 'test', - severity: 1, - confidence: 1, - report_type: 1 - ) - end - - let(:merge_request) do - merge_requests.create!( - target_project_id: project.id, - source_branch: "other", - target_branch: "main", - author_id: user.id, - title: 'Feedback Merge Request' - ) - end - - let!(:vulnerability_read) do - vulnerability_reads.create!( - project_id: project.id, - vulnerability_id: vulnerability.id, - scanner_id: scanner.id, - severity: 1, - report_type: 1, - state: 1, - uuid: SecureRandom.uuid - ) - end - - let!(:merge_request_link) do - merge_request_links.create!( - vulnerability_id: vulnerability.id, merge_request_id: merge_request.id) - end - - subject(:perform_migration) do - described_class.new( - start_id: vulnerability_reads.first.vulnerability_id, - end_id: vulnerability_reads.last.vulnerability_id, - batch_table: :vulnerability_reads, - batch_column: :vulnerability_id, - sub_batch_size: vulnerability_reads.count, - pause_ms: 0, - connection: ActiveRecord::Base.connection - ).perform - end - - before do - # Unset since the trigger already sets during merge_request_link creation. - vulnerability_reads.update_all(has_merge_request: false) - end - - it 'sets the has_merge_request of existing record' do - expect { perform_migration }.to change { vulnerability_read.reload.has_merge_request }.from(false).to(true) - end - - it 'does not modify has_merge_request of other vulnerabilities which do not have merge request' do - vulnerability_2 = vulnerabilities.create!( - project_id: project.id, - author_id: user.id, - title: 'test 2', - severity: 1, - confidence: 1, - report_type: 1 - ) - - vulnerability_read_2 = vulnerability_reads.create!( - project_id: project.id, - vulnerability_id: vulnerability_2.id, - scanner_id: scanner.id, - severity: 1, - report_type: 1, - state: 1, - uuid: SecureRandom.uuid - ) - - expect { perform_migration }.not_to change { vulnerability_read_2.reload.has_merge_request }.from(false) - end -end diff --git a/spec/migrations/20230907155247_queue_backfill_has_merge_request_of_vulnerability_reads_spec.rb b/spec/migrations/20230907155247_queue_backfill_has_merge_request_of_vulnerability_reads_spec.rb deleted file mode 100644 index 519e81cc933b969cf16a10c2e94c4019b07fa52d..0000000000000000000000000000000000000000 --- a/spec/migrations/20230907155247_queue_backfill_has_merge_request_of_vulnerability_reads_spec.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require_migration! - -RSpec.describe QueueBackfillHasMergeRequestOfVulnerabilityReads, feature_category: :database do - let!(:batched_migration) { described_class::MIGRATION } - - it 'schedules a new batched migration' do - reversible_migration do |migration| - migration.before -> { - expect(batched_migration).not_to have_scheduled_batched_migration - } - - migration.after -> { - expect(batched_migration).to have_scheduled_batched_migration( - table_name: :vulnerability_reads, - column_name: :vulnerability_id, - interval: described_class::DELAY_INTERVAL, - batch_size: described_class::BATCH_SIZE, - sub_batch_size: described_class::SUB_BATCH_SIZE - ) - } - end - end -end