diff --git a/.rubocop_todo/gitlab/strong_memoize_attr.yml b/.rubocop_todo/gitlab/strong_memoize_attr.yml index 181b18807de6ba88f18416fe2b4e5adcd3cfcd94..8cf4cf5655d8510dccc54fcce375cb5ae5e0c3bf 100644 --- a/.rubocop_todo/gitlab/strong_memoize_attr.yml +++ b/.rubocop_todo/gitlab/strong_memoize_attr.yml @@ -366,7 +366,6 @@ Gitlab/StrongMemoizeAttr: - 'ee/lib/ee/gitlab/analytics/cycle_analytics/stage_events.rb' - 'ee/lib/ee/gitlab/auth/o_auth/auth_hash.rb' - 'ee/lib/ee/gitlab/background_migration/migrate_approver_to_approval_rules.rb' - - 'ee/lib/ee/gitlab/background_migration/populate_resolved_on_default_branch_column.rb' - 'ee/lib/ee/gitlab/checks/base_checker.rb' - 'ee/lib/ee/gitlab/checks/diff_check.rb' - 'ee/lib/ee/gitlab/ci/matching/runner_matcher.rb' diff --git a/.rubocop_todo/layout/line_length.yml b/.rubocop_todo/layout/line_length.yml index bd6418c30a66ea759ad08c4afdc32d208b9e47e9..3f1210267cb41e2694cde63eeb90086d282f2e0d 100644 --- a/.rubocop_todo/layout/line_length.yml +++ b/.rubocop_todo/layout/line_length.yml @@ -820,7 +820,6 @@ Layout/LineLength: - 'ee/app/models/vulnerabilities/finding.rb' - 'ee/app/models/vulnerabilities/finding_identifier.rb' - 'ee/app/models/vulnerabilities/finding_link.rb' - - 'ee/app/models/vulnerabilities/finding_pipeline.rb' - 'ee/app/models/vulnerabilities/finding_remediation.rb' - 'ee/app/models/vulnerabilities/flag.rb' - 'ee/app/models/vulnerabilities/historical_statistic.rb' @@ -1056,7 +1055,6 @@ Layout/LineLength: - 'ee/lib/ee/gitlab/background_migration/create_security_setting.rb' - 'ee/lib/ee/gitlab/background_migration/migrate_approver_to_approval_rules.rb' - 'ee/lib/ee/gitlab/background_migration/populate_latest_pipeline_ids.rb' - - 'ee/lib/ee/gitlab/background_migration/populate_resolved_on_default_branch_column.rb' - 'ee/lib/ee/gitlab/checks/push_rules/commit_check.rb' - 'ee/lib/ee/gitlab/checks/push_rules/file_size_check.rb' - 'ee/lib/ee/gitlab/ci/pipeline/chain/create_cross_database_associations.rb' @@ -2281,7 +2279,6 @@ Layout/LineLength: - 'lib/gitlab/background_migration/migrate_approver_to_approval_rules_in_batch.rb' - 'lib/gitlab/background_migration/migrate_job_artifact_registry_to_ssf.rb' - 'lib/gitlab/background_migration/migrate_requirements_to_work_items.rb' - - 'lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb' - 'lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb' - 'lib/gitlab/bullet/exclusions.rb' - 'lib/gitlab/cache/helpers.rb' diff --git a/.rubocop_todo/lint/redundant_cop_disable_directive.yml b/.rubocop_todo/lint/redundant_cop_disable_directive.yml index 7b01d9c4ff16be25b1694c37ae3d54dec854dca2..a9274e7b35fcf79a0403c45d33e633a1f524ce3b 100644 --- a/.rubocop_todo/lint/redundant_cop_disable_directive.yml +++ b/.rubocop_todo/lint/redundant_cop_disable_directive.yml @@ -293,7 +293,6 @@ Lint/RedundantCopDisableDirective: - 'lib/gitlab/background_migration/migrate_vulnerabilities_feedback_to_vulnerabilities_state_transition.rb' - 'lib/gitlab/background_migration/populate_denormalized_columns_for_sbom_occurrences.rb' - 'lib/gitlab/background_migration/populate_latest_pipeline_ids.rb' - - 'lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb' - 'lib/gitlab/background_migration/populate_vulnerability_dismissal_fields.rb' - 'lib/gitlab/background_migration/purge_stale_security_scans.rb' - 'lib/gitlab/background_migration/resolve_vulnerabilities_for_removed_analyzers.rb' diff --git a/.rubocop_todo/migration/background_migration_record.yml b/.rubocop_todo/migration/background_migration_record.yml index bf8e962dcab3e3fbeebbb230c800ac513f2d8f58..af93c19f11f775f9641a7760b606bb0487d06b02 100644 --- a/.rubocop_todo/migration/background_migration_record.yml +++ b/.rubocop_todo/migration/background_migration_record.yml @@ -7,7 +7,6 @@ Migration/BackgroundMigrationRecord: - 'ee/lib/ee/gitlab/background_migration/migrate_approver_to_approval_rules_in_batch.rb' - 'ee/lib/ee/gitlab/background_migration/migrate_requirements_to_work_items.rb' - 'ee/lib/ee/gitlab/background_migration/populate_latest_pipeline_ids.rb' - - 'ee/lib/ee/gitlab/background_migration/populate_resolved_on_default_branch_column.rb' - 'lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb' - 'lib/gitlab/background_migration/backfill_project_repositories.rb' - 'lib/gitlab/background_migration/backfill_topics_title.rb' diff --git a/.rubocop_todo/migration/batched_migration_base_class.yml b/.rubocop_todo/migration/batched_migration_base_class.yml index 5f3b072c051d1b70b24fab04b1cccef87a7e8c93..a674bab35bc9d982d1d1120f657d42bcc9ad3f9b 100644 --- a/.rubocop_todo/migration/batched_migration_base_class.yml +++ b/.rubocop_todo/migration/batched_migration_base_class.yml @@ -26,7 +26,6 @@ Migration/BatchedMigrationBaseClass: - 'lib/gitlab/background_migration/migrate_null_private_profile_to_false.rb' - 'lib/gitlab/background_migration/migrate_requirements_to_work_items.rb' - 'lib/gitlab/background_migration/populate_latest_pipeline_ids.rb' - - 'lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb' - 'lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb' - 'lib/gitlab/background_migration/project_namespaces/models/namespace.rb' - 'lib/gitlab/background_migration/project_namespaces/models/project.rb' diff --git a/.rubocop_todo/rspec/context_wording.yml b/.rubocop_todo/rspec/context_wording.yml index a34756df00d5b91f7db34589fc04f96b20a1f6d5..74e659e053b0a587c8349d150b45913e2da0264b 100644 --- a/.rubocop_todo/rspec/context_wording.yml +++ b/.rubocop_todo/rspec/context_wording.yml @@ -461,7 +461,6 @@ RSpec/ContextWording: - 'ee/spec/models/saml_provider_spec.rb' - 'ee/spec/models/status_page/project_setting_spec.rb' - 'ee/spec/models/vulnerabilities/feedback_spec.rb' - - 'ee/spec/models/vulnerabilities/finding_pipeline_spec.rb' - 'ee/spec/models/vulnerabilities/finding_spec.rb' - 'ee/spec/models/vulnerabilities/statistic_spec.rb' - 'ee/spec/policies/app_sec/fuzzing/coverage/corpus_policy_spec.rb' diff --git a/.rubocop_todo/style/inline_disable_annotation.yml b/.rubocop_todo/style/inline_disable_annotation.yml index 59323faefec2d3fecdb41cf3ee7b2d703c591c0a..1123966d7a9c3a079b753ac14ccb5b57c3969ec0 100644 --- a/.rubocop_todo/style/inline_disable_annotation.yml +++ b/.rubocop_todo/style/inline_disable_annotation.yml @@ -2019,7 +2019,6 @@ Style/InlineDisableAnnotation: - 'lib/gitlab/background_migration/migrate_shared_vulnerability_scanners.rb' - 'lib/gitlab/background_migration/populate_denormalized_columns_for_sbom_occurrences.rb' - 'lib/gitlab/background_migration/populate_latest_pipeline_ids.rb' - - 'lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb' - 'lib/gitlab/background_migration/populate_vulnerability_dismissal_fields.rb' - 'lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb' - 'lib/gitlab/background_migration/purge_stale_security_scans.rb' diff --git a/.rubocop_todo/style/string_concatenation.yml b/.rubocop_todo/style/string_concatenation.yml index a5ea884b5f687668641f53e8dfa95fb1daffaecf..d84850fcb5d47509dc9b2cd505f847d2def8c958 100644 --- a/.rubocop_todo/style/string_concatenation.yml +++ b/.rubocop_todo/style/string_concatenation.yml @@ -43,7 +43,6 @@ Style/StringConcatenation: - 'ee/lib/api/project_mirror.rb' - 'ee/lib/ee/gitlab/auth/ldap/person.rb' - 'ee/lib/ee/gitlab/background_migration/populate_latest_pipeline_ids.rb' - - 'ee/lib/ee/gitlab/background_migration/populate_resolved_on_default_branch_column.rb' - 'ee/lib/gitlab/geo/git_ssh_proxy.rb' - 'ee/lib/omni_auth/strategies/kerberos.rb' - 'ee/lib/tasks/gitlab/license.rake' diff --git a/config/gitlab_loose_foreign_keys.yml b/config/gitlab_loose_foreign_keys.yml index fdaf8fcac3b0ae13d792c66bea2f2296c8cba6dd..6b7afeb92e8deadae8bc193a5c3480b74f780f24 100644 --- a/config/gitlab_loose_foreign_keys.yml +++ b/config/gitlab_loose_foreign_keys.yml @@ -650,10 +650,6 @@ vulnerability_namespace_historical_statistics: - table: namespaces column: namespace_id on_delete: async_delete -vulnerability_occurrence_pipelines: - - table: ci_pipelines - column: pipeline_id - on_delete: async_delete vulnerability_occurrences: - table: ci_pipelines column: initial_pipeline_id diff --git a/db/docs/batched_background_migrations/backfill_vulnerability_occurrence_pipelines_project_id.yml b/db/docs/batched_background_migrations/backfill_vulnerability_occurrence_pipelines_project_id.yml deleted file mode 100644 index 5e77f5517f3d1c5c063256d9a4c0e62d94f8f67e..0000000000000000000000000000000000000000 --- a/db/docs/batched_background_migrations/backfill_vulnerability_occurrence_pipelines_project_id.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -migration_job_name: BackfillVulnerabilityOccurrencePipelinesProjectId -description: Backfills sharding key `vulnerability_occurrence_pipelines.project_id` from `vulnerability_occurrences`. -feature_category: vulnerability_management -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/161221 -milestone: '17.3' -queued_migration_version: 20240730172818 -finalized_by: # version of the migration that finalized this BBM diff --git a/db/docs/deleted_tables/vulnerability_occurrence_pipelines.yml b/db/docs/deleted_tables/vulnerability_occurrence_pipelines.yml new file mode 100644 index 0000000000000000000000000000000000000000..b1414c2aa51bebc51d55c275adc16b738a9c04d3 --- /dev/null +++ b/db/docs/deleted_tables/vulnerability_occurrence_pipelines.yml @@ -0,0 +1,12 @@ +--- +table_name: vulnerability_occurrence_pipelines +classes: +- Vulnerabilities::FindingPipeline +feature_categories: +- vulnerability_management +description: Join table between Findings and Pipelines +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/7578 +milestone: '11.5' +gitlab_schema: gitlab_sec +removed_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/170153 +removed_in_milestone: '17.6' diff --git a/db/docs/vulnerability_occurrence_pipelines.yml b/db/docs/vulnerability_occurrence_pipelines.yml deleted file mode 100644 index 2fb440db036b8b150a3173fc502543e99a11669b..0000000000000000000000000000000000000000 --- a/db/docs/vulnerability_occurrence_pipelines.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -table_name: vulnerability_occurrence_pipelines -classes: -- Vulnerabilities::FindingPipeline -feature_categories: -- vulnerability_management -description: Join table between Findings and Pipelines -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/7578 -milestone: '11.5' -gitlab_schema: gitlab_sec -desired_sharding_key: - project_id: - references: projects - backfill_via: - parent: - foreign_key: occurrence_id - table: vulnerability_occurrences - sharding_key: project_id - belongs_to: finding -desired_sharding_key_migration_job_name: BackfillVulnerabilityOccurrencePipelinesProjectId diff --git a/db/gitlab_schemas/gitlab_sec.yaml b/db/gitlab_schemas/gitlab_sec.yaml index 8b071d7b6632e1ec9c1addc700e9b0fb5229bbe6..7b699c83d0e1b57030f48a8bffa24b01e4ddc693 100644 --- a/db/gitlab_schemas/gitlab_sec.yaml +++ b/db/gitlab_schemas/gitlab_sec.yaml @@ -25,7 +25,6 @@ allow_cross_joins: - vulnerability_issue_links - vulnerability_merge_request_links - vulnerability_occurrence_identifiers - - vulnerability_occurrence_pipelines - vulnerability_occurrences - vulnerability_reads - vulnerability_remediations @@ -77,7 +76,6 @@ allow_cross_transactions: - vulnerability_issue_links - vulnerability_merge_request_links - vulnerability_occurrence_identifiers - - vulnerability_occurrence_pipelines - vulnerability_occurrences - vulnerability_reads - vulnerability_remediations @@ -127,7 +125,6 @@ allow_cross_foreign_keys: - vulnerability_issue_links - vulnerability_merge_request_links - vulnerability_occurrence_identifiers - - vulnerability_occurrence_pipelines - vulnerability_occurrences - vulnerability_reads - vulnerability_remediations diff --git a/db/post_migrate/20240730172818_queue_backfill_vulnerability_occurrence_pipelines_project_id.rb b/db/post_migrate/20240730172818_queue_backfill_vulnerability_occurrence_pipelines_project_id.rb index 251639d058fcbd51b2557c4426d97ee165075ac6..ae9bf7f6fc283f73e22c35366a638e47070e72e8 100644 --- a/db/post_migrate/20240730172818_queue_backfill_vulnerability_occurrence_pipelines_project_id.rb +++ b/db/post_migrate/20240730172818_queue_backfill_vulnerability_occurrence_pipelines_project_id.rb @@ -5,23 +5,9 @@ class QueueBackfillVulnerabilityOccurrencePipelinesProjectId < Gitlab::Database: restrict_gitlab_migration gitlab_schema: :gitlab_sec MIGRATION = "BackfillVulnerabilityOccurrencePipelinesProjectId" - DELAY_INTERVAL = 2.minutes - BATCH_SIZE = 1000 - SUB_BATCH_SIZE = 100 def up - queue_batched_background_migration( - MIGRATION, - :vulnerability_occurrence_pipelines, - :id, - :project_id, - :vulnerability_occurrences, - :project_id, - :occurrence_id, - job_interval: DELAY_INTERVAL, - batch_size: BATCH_SIZE, - sub_batch_size: SUB_BATCH_SIZE - ) + # This migration has been deleted because the table has been dropped. end def down diff --git a/db/post_migrate/20241022181600_remove_backfill_vulnerability_occurrence_pipelines_project_id.rb b/db/post_migrate/20241022181600_remove_backfill_vulnerability_occurrence_pipelines_project_id.rb new file mode 100644 index 0000000000000000000000000000000000000000..3a30ce9b06784918b5b7758500cebfacebd4320f --- /dev/null +++ b/db/post_migrate/20241022181600_remove_backfill_vulnerability_occurrence_pipelines_project_id.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class RemoveBackfillVulnerabilityOccurrencePipelinesProjectId < Gitlab::Database::Migration[2.2] + milestone '17.6' + restrict_gitlab_migration gitlab_schema: :gitlab_sec + + MIGRATION = "BackfillVulnerabilityOccurrencePipelinesProjectId" + + def up + delete_batched_background_migration( + MIGRATION, + :vulnerability_occurrence_pipelines, + :id, + [ + :project_id, + :vulnerability_occurrences, + :project_id, + :occurrence_id + ] + ) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20241022181639_drop_vulnerability_occurrence_pipelines.rb b/db/post_migrate/20241022181639_drop_vulnerability_occurrence_pipelines.rb new file mode 100644 index 0000000000000000000000000000000000000000..e5a4ded1c7079ff2c79c7f65f6a7e860bc4100db --- /dev/null +++ b/db/post_migrate/20241022181639_drop_vulnerability_occurrence_pipelines.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +class DropVulnerabilityOccurrencePipelines < Gitlab::Database::Migration[2.2] + include Gitlab::Database::SchemaHelpers + + TABLE_NAME = :vulnerability_occurrence_pipelines + TRIGGER_NAME = 'trigger_8ba31bddd655' + + milestone '17.6' + + disable_ddl_transaction! + + def up + drop_table TABLE_NAME + drop_function TRIGGER_NAME + end + + def down + create_table TABLE_NAME do |t| + t.timestamps_with_timezone null: false + t.bigint :occurrence_id, null: false + t.bigint :pipeline_id, null: false + t.bigint :project_id + end + + add_concurrent_index TABLE_NAME, [:occurrence_id, :pipeline_id], + unique: true, + name: 'vulnerability_occurrence_pipelines_on_unique_keys' + add_concurrent_index TABLE_NAME, [:occurrence_id, :id], + order: { occurrence_id: :asc, id: :desc }, + name: 'index_vulnerability_occurrence_pipelines_occurrence_id_and_id' + add_concurrent_index TABLE_NAME, :pipeline_id, + name: 'index_vulnerability_occurrence_pipelines_on_pipeline_id' + + add_concurrent_foreign_key TABLE_NAME, :vulnerability_occurrences, + column: :occurrence_id, on_delete: :cascade, name: 'fk_rails_dc3ae04693' + + install_sharding_key_assignment_trigger( + table: TABLE_NAME, + sharding_key: :project_id, + parent_table: :vulnerability_occurrences, + parent_sharding_key: :project_id, + foreign_key: :occurrence_id, + trigger_name: TRIGGER_NAME + ) + end +end diff --git a/db/schema_migrations/20241022181600 b/db/schema_migrations/20241022181600 new file mode 100644 index 0000000000000000000000000000000000000000..bf22610835056e0c910b2295c1acb12619be50c6 --- /dev/null +++ b/db/schema_migrations/20241022181600 @@ -0,0 +1 @@ +50de83ee644af67af6dcac2dd533fa03e20249fd9056418b038d1889599b6bd8 \ No newline at end of file diff --git a/db/schema_migrations/20241022181639 b/db/schema_migrations/20241022181639 new file mode 100644 index 0000000000000000000000000000000000000000..419300f88d08fe924b21f2cab12602881fd8bdc8 --- /dev/null +++ b/db/schema_migrations/20241022181639 @@ -0,0 +1 @@ +6b16fc7572186fdf3ef1fe1d3161d49e9ff46c2ffa927e1a6fdf07834080bc2c \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 4a30f71089fcd37c0b160f45d997d8cd855e03e2..9d2c355bafe287012109bc8671b7b5fccbce4b53 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -1899,22 +1899,6 @@ RETURN NEW; END $$; -CREATE FUNCTION trigger_8ba31bddd655() RETURNS trigger - LANGUAGE plpgsql - AS $$ -BEGIN -IF NEW."project_id" IS NULL THEN - SELECT "project_id" - INTO NEW."project_id" - FROM "vulnerability_occurrences" - WHERE "vulnerability_occurrences"."id" = NEW."occurrence_id"; -END IF; - -RETURN NEW; - -END -$$; - CREATE FUNCTION trigger_8d002f38bdef() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -20781,24 +20765,6 @@ CREATE SEQUENCE vulnerability_occurrence_identifiers_id_seq ALTER SEQUENCE vulnerability_occurrence_identifiers_id_seq OWNED BY vulnerability_occurrence_identifiers.id; -CREATE TABLE vulnerability_occurrence_pipelines ( - id bigint NOT NULL, - created_at timestamp with time zone NOT NULL, - updated_at timestamp with time zone NOT NULL, - occurrence_id bigint NOT NULL, - pipeline_id bigint NOT NULL, - project_id bigint -); - -CREATE SEQUENCE vulnerability_occurrence_pipelines_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - -ALTER SEQUENCE vulnerability_occurrence_pipelines_id_seq OWNED BY vulnerability_occurrence_pipelines.id; - CREATE TABLE vulnerability_occurrences ( id bigint NOT NULL, created_at timestamp with time zone NOT NULL, @@ -23248,8 +23214,6 @@ ALTER TABLE ONLY vulnerability_namespace_historical_statistics ALTER COLUMN id S ALTER TABLE ONLY vulnerability_occurrence_identifiers ALTER COLUMN id SET DEFAULT nextval('vulnerability_occurrence_identifiers_id_seq'::regclass); -ALTER TABLE ONLY vulnerability_occurrence_pipelines ALTER COLUMN id SET DEFAULT nextval('vulnerability_occurrence_pipelines_id_seq'::regclass); - ALTER TABLE ONLY vulnerability_occurrences ALTER COLUMN id SET DEFAULT nextval('vulnerability_occurrences_id_seq'::regclass); ALTER TABLE ONLY vulnerability_reads ALTER COLUMN id SET DEFAULT nextval('vulnerability_reads_id_seq'::regclass); @@ -26048,9 +26012,6 @@ ALTER TABLE ONLY vulnerability_namespace_historical_statistics ALTER TABLE ONLY vulnerability_occurrence_identifiers ADD CONSTRAINT vulnerability_occurrence_identifiers_pkey PRIMARY KEY (id); -ALTER TABLE ONLY vulnerability_occurrence_pipelines - ADD CONSTRAINT vulnerability_occurrence_pipelines_pkey PRIMARY KEY (id); - ALTER TABLE ONLY vulnerability_occurrences ADD CONSTRAINT vulnerability_occurrences_pkey PRIMARY KEY (id); @@ -31688,10 +31649,6 @@ CREATE INDEX index_vulnerability_occurrence_identifiers_on_identifier_id ON vuln CREATE UNIQUE INDEX index_vulnerability_occurrence_identifiers_on_unique_keys ON vulnerability_occurrence_identifiers USING btree (occurrence_id, identifier_id); -CREATE INDEX index_vulnerability_occurrence_pipelines_occurrence_id_and_id ON vulnerability_occurrence_pipelines USING btree (occurrence_id, id DESC); - -CREATE INDEX index_vulnerability_occurrence_pipelines_on_pipeline_id ON vulnerability_occurrence_pipelines USING btree (pipeline_id); - CREATE INDEX index_vulnerability_occurrences_for_override_uuids_logic ON vulnerability_occurrences USING btree (project_id, report_type, location_fingerprint); CREATE INDEX index_vulnerability_occurrences_on_initial_pipeline_id ON vulnerability_occurrences USING btree (initial_pipeline_id); @@ -32168,8 +32125,6 @@ CREATE UNIQUE INDEX virtual_reg_packages_maven_reg_upstreams_on_unique_upstream_ CREATE UNIQUE INDEX virtual_registries_pkgs_maven_registries_on_unique_group_ids ON virtual_registries_packages_maven_registries USING btree (group_id); -CREATE UNIQUE INDEX vulnerability_occurrence_pipelines_on_unique_keys ON vulnerability_occurrence_pipelines USING btree (occurrence_id, pipeline_id); - CREATE INDEX wi_colors_namespace_id_index ON work_item_colors USING btree (namespace_id); CREATE INDEX wi_datessources_due_date_sourcing_milestone_id_index ON work_item_dates_sources USING btree (due_date_sourcing_milestone_id); @@ -33944,8 +33899,6 @@ CREATE TRIGGER trigger_8ac78f164b2d BEFORE INSERT OR UPDATE ON design_management CREATE TRIGGER trigger_8b39d532224c BEFORE INSERT OR UPDATE ON ci_secure_file_states FOR EACH ROW EXECUTE FUNCTION trigger_8b39d532224c(); -CREATE TRIGGER trigger_8ba31bddd655 BEFORE INSERT OR UPDATE ON vulnerability_occurrence_pipelines FOR EACH ROW EXECUTE FUNCTION trigger_8ba31bddd655(); - CREATE TRIGGER trigger_8d002f38bdef BEFORE INSERT OR UPDATE ON packages_debian_group_components FOR EACH ROW EXECUTE FUNCTION trigger_8d002f38bdef(); CREATE TRIGGER trigger_8d17725116fe BEFORE INSERT OR UPDATE ON merge_request_reviewers FOR EACH ROW EXECUTE FUNCTION trigger_8d17725116fe(); @@ -37324,9 +37277,6 @@ ALTER TABLE ONLY dependency_proxy_blobs ALTER TABLE ONLY board_user_preferences ADD CONSTRAINT fk_rails_dbebdaa8fe FOREIGN KEY (board_id) REFERENCES boards(id) ON DELETE CASCADE; -ALTER TABLE ONLY vulnerability_occurrence_pipelines - ADD CONSTRAINT fk_rails_dc3ae04693 FOREIGN KEY (occurrence_id) REFERENCES vulnerability_occurrences(id) ON DELETE CASCADE; - ALTER TABLE ONLY instance_audit_events_streaming_headers ADD CONSTRAINT fk_rails_dc933c1f3c FOREIGN KEY (instance_external_audit_event_destination_id) REFERENCES audit_events_instance_external_audit_event_destinations(id) ON DELETE CASCADE; diff --git a/ee/app/models/vulnerabilities/finding.rb b/ee/app/models/vulnerabilities/finding.rb index 12f04544331d70a466161988a148e3ca2aca6d7f..00f04fa167fd3fab2f7e646e95c2b1ddbc5a5a50 100644 --- a/ee/app/models/vulnerabilities/finding.rb +++ b/ee/app/models/vulnerabilities/finding.rb @@ -93,8 +93,6 @@ class Finding < Gitlab::Database::SecApplicationRecord has_many :finding_remediations, class_name: 'Vulnerabilities::FindingRemediation', inverse_of: :finding, foreign_key: 'vulnerability_occurrence_id' has_many :remediations, through: :finding_remediations - has_many :finding_pipelines, class_name: 'Vulnerabilities::FindingPipeline', inverse_of: :finding, foreign_key: 'occurrence_id' - # rubocop: disable Rails/InverseOf -- these relations are not present on Ci::Pipeline belongs_to :initial_finding_pipeline, class_name: '::Ci::Pipeline', foreign_key: 'initial_pipeline_id' belongs_to :latest_finding_pipeline, class_name: '::Ci::Pipeline', foreign_key: 'latest_pipeline_id' diff --git a/ee/app/models/vulnerabilities/finding_pipeline.rb b/ee/app/models/vulnerabilities/finding_pipeline.rb deleted file mode 100644 index b35605b28f58a11afa47bd173d65196a4b3e6a19..0000000000000000000000000000000000000000 --- a/ee/app/models/vulnerabilities/finding_pipeline.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -module Vulnerabilities - class FindingPipeline < Gitlab::Database::SecApplicationRecord - include EachBatch - - self.table_name = "vulnerability_occurrence_pipelines" - - alias_attribute :finding_id, :occurrence_id - - belongs_to :finding, class_name: 'Vulnerabilities::Finding', inverse_of: :finding_pipelines, foreign_key: 'occurrence_id' - belongs_to :pipeline, class_name: '::Ci::Pipeline' - - validates :finding, presence: true - validates :pipeline, presence: true - validates :pipeline_id, uniqueness: { scope: [:occurrence_id] } - - scope :by_finding_id, ->(finding_ids) { where(occurrence_id: finding_ids) } - end -end diff --git a/ee/lib/ee/gitlab/background_migration/populate_resolved_on_default_branch_column.rb b/ee/lib/ee/gitlab/background_migration/populate_resolved_on_default_branch_column.rb deleted file mode 100644 index 4422461a800f168c45e70df50fc74d3472c2fa27..0000000000000000000000000000000000000000 --- a/ee/lib/ee/gitlab/background_migration/populate_resolved_on_default_branch_column.rb +++ /dev/null @@ -1,338 +0,0 @@ -# frozen_string_literal: true - -module EE - module Gitlab - module BackgroundMigration - module PopulateResolvedOnDefaultBranchColumn - def perform(*project_ids) - project_ids.flatten.each { |project_id| PopulateResolvedOnDefaultBranchColumnForProject.perform(project_id) } - end - - module Routable - extend ActiveSupport::Concern - - included do - has_one :route, as: :source - end - - def full_path - route&.path || build_full_path - end - - def build_full_path - if parent && path - parent.full_path + '/' + path - else - path - end - end - end - - module Visibility - PUBLIC_LEVEL = 20 - - def public? - visibility_level == PUBLIC_LEVEL - end - end - - # This class depends on Gitlab::CurrentSettings - class Project < ActiveRecord::Base - include Routable - include Visibility - include ::Gitlab::Utils::StrongMemoize - - self.table_name = 'projects' - - # These are the artifact file types to query - # only security report related artifacts. - # sast: 5 - # dependency_scanning: 6 - # container_scanning: 7 - # dast: 8 - # secret_detection: 21 - # coverage_fuzzing: 23 - FILE_TYPES = [5, 6, 7, 8, 21, 23].freeze - LATEST_PIPELINE_WITH_REPORTS_SQL = <<~SQL - SELECT - "ci_pipelines"."id" - FROM - "ci_pipelines" - WHERE - ("ci_pipelines"."id" IN ( - SELECT - "ci_pipelines"."id" - FROM - "ci_pipelines" - WHERE - ci_pipelines.project_id = %{project_id} - AND ci_pipelines.ref = %{ref} - AND ci_pipelines.status IN ('success') - ORDER BY - "ci_pipelines"."id" DESC - LIMIT 100)) - AND (EXISTS ( - SELECT - 1 - FROM - "ci_builds" - WHERE - "ci_builds"."type" = 'Ci::Build' - AND ("ci_builds"."retried" IS FALSE OR "ci_builds"."retried" IS NULL) - AND (EXISTS ( - SELECT - 1 - FROM - "ci_job_artifacts" - WHERE - (ci_builds.id = ci_job_artifacts.job_id) - AND "ci_job_artifacts"."file_type" IN (%{file_types}))) - AND (ci_pipelines.id = ci_builds.commit_id))) - ORDER BY - "ci_pipelines"."id" DESC - LIMIT 1 - SQL - - belongs_to :namespace - alias_method :parent, :namespace - - has_one :route, as: :source - has_many :vulnerabilities - - def self.polymorphic_name - 'Project' - end - - def resolved_vulnerabilities - return Vulnerability.none unless latest_pipeline_id - - vulnerabilities.not_found_in_pipeline_id(latest_pipeline_id) - end - - private - - delegate :connection, to: :'self.class', private: true - - def latest_pipeline_id - strong_memoize(:latest_pipeline_id) { pipeline_with_reports&.fetch('id') } - end - - def pipeline_with_reports - connection.execute(pipeline_with_reports_sql).first - end - - def pipeline_with_reports_sql - format(LATEST_PIPELINE_WITH_REPORTS_SQL, project_id: id, ref: connection.quote(default_branch), file_types: FILE_TYPES.join(', ')) - end - - def default_branch - @default_branch ||= repository.root_ref || default_branch_from_preferences - end - - def repository - @repository ||= Repository.new(full_path, self, shard: repository_storage, disk_path: storage.disk_path) - end - - def storage - @storage ||= - if hashed_repository_storage? - Storage::Hashed.new(self) - else - Storage::LegacyProject.new(self) - end - end - - def hashed_repository_storage? - storage_version.to_i >= 1 - end - - def default_branch_from_preferences - ::Gitlab::CurrentSettings.default_branch_name if repository.empty? - end - end - - module Storage - class Hashed - attr_accessor :container - - REPOSITORY_PATH_PREFIX = '@hashed' - - def initialize(container) - @container = container - end - - def base_dir - "#{REPOSITORY_PATH_PREFIX}/#{disk_hash[0..1]}/#{disk_hash[2..3]}" if disk_hash - end - - def disk_path - "#{base_dir}/#{disk_hash}" if disk_hash - end - - private - - def disk_hash - @disk_hash ||= Digest::SHA2.hexdigest(container.id.to_s) if container.id - end - end - - class LegacyProject - attr_accessor :project - - def initialize(project) - @project = project - end - - def disk_path - project.full_path - end - end - end - - class Namespace < ActiveRecord::Base - include Routable - include Visibility - - self.table_name = 'namespaces' - - belongs_to :parent, class_name: 'Namespace' - - def self.find_sti_class(type_name) - super("EE::Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn::#{type_name}") - end - end - - class Group < Namespace - def self.polymorphic_name - 'Group' - end - end - - class Route < ActiveRecord::Base - self.table_name = 'routes' - end - - class Vulnerability < ActiveRecord::Base - include EachBatch - - self.table_name = 'vulnerabilities' - - scope :not_found_in_pipeline_id, ->(pipeline_id) do - where(<<~SQL) - NOT EXISTS ( - SELECT 1 - FROM vulnerability_occurrences vo - INNER JOIN vulnerability_occurrence_pipelines vop ON vop.occurrence_id = vo.id - WHERE vo.vulnerability_id = vulnerabilities.id AND vop.pipeline_id = #{pipeline_id} - ) - SQL - end - end - - # This class depends on following classes - # GlRepository class defined in `lib/gitlab/gl_repository.rb` - # Repository class defined in `lib/gitlab/git/repository.rb`. - class Repository - def initialize(full_path, container, shard:, disk_path: nil, repo_type: ::Gitlab::GlRepository::PROJECT) - @full_path = full_path - @shard = shard - @disk_path = disk_path || full_path - @container = container - @commit_cache = {} - @repo_type = repo_type - end - - def root_ref - raw_repository&.root_ref - rescue Gitlab::Git::Repository::NoRepository - end - - def empty? - return true unless exists? - - !has_visible_content? - end - - private - - attr_reader :full_path, :shard, :disk_path, :container, :repo_type - - delegate :has_visible_content?, to: :raw_repository, private: true - - def exists? - return false unless full_path - - raw_repository.exists? - end - - def raw_repository - return unless full_path - - @raw_repository ||= initialize_raw_repository - end - - def initialize_raw_repository - ::Gitlab::Git::Repository.new( - shard, - disk_path + '.git', - repo_type.identifier_for_container(container), - container.full_path - ) - end - end - - class PopulateResolvedOnDefaultBranchColumnForProject - def self.perform(project_id) - new(project_id).perform - end - - def initialize(project_id) - self.project_id = project_id - self.updated_count = 0 - end - - def perform - update_vulnerabilities - log_info - rescue StandardError => e - log_error(e) - end - - attr_accessor :project_id, :updated_count - - private - - def update_vulnerabilities - return if project.resolved_vulnerabilities.none? - - project.vulnerabilities.each_batch(of: 100) do |relation| - self.updated_count += relation.merge(project.resolved_vulnerabilities) - .update_all(resolved_on_default_branch: true) - end - end - - def log_info - ::Gitlab::BackgroundMigration::Logger.info( - migrator: 'PopulateResolvedOnDefaultBranchColumnForProject', - message: 'Project migrated', - updated_count: updated_count, - project_id: project_id - ) - end - - def log_error(error) - ::Gitlab::BackgroundMigration::Logger.error( - migrator: 'PopulateResolvedOnDefaultBranchColumnForProject', - message: error.message, - project_id: project_id - ) - end - - def project - @project ||= Project.find(project_id) - end - end - end - end - end -end diff --git a/ee/lib/gitlab/ingestion/bulk_insertable_task.rb b/ee/lib/gitlab/ingestion/bulk_insertable_task.rb index 8367001593b3f9569c965b5292e2f9b9be3094dc..bb8d34f9eb6594c820d706b803438ff7a07b8df6 100644 --- a/ee/lib/gitlab/ingestion/bulk_insertable_task.rb +++ b/ee/lib/gitlab/ingestion/bulk_insertable_task.rb @@ -87,7 +87,6 @@ def return_data vulnerability_finding_links vulnerability_identifiers vulnerability_occurrence_identifiers - vulnerability_occurrence_pipelines vulnerability_occurrences ], url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/474635' diff --git a/ee/spec/factories/vulnerabilities/finding_pipelines.rb b/ee/spec/factories/vulnerabilities/finding_pipelines.rb deleted file mode 100644 index 08ee2bc9ae528163e172f4e9f3d680a05baa1819..0000000000000000000000000000000000000000 --- a/ee/spec/factories/vulnerabilities/finding_pipelines.rb +++ /dev/null @@ -1,8 +0,0 @@ -# frozen_string_literal: true - -FactoryBot.define do - factory :vulnerabilities_finding_pipeline, class: 'Vulnerabilities::FindingPipeline' do - finding factory: :vulnerabilities_finding - pipeline factory: :ci_pipeline - end -end diff --git a/ee/spec/models/vulnerabilities/finding_pipeline_spec.rb b/ee/spec/models/vulnerabilities/finding_pipeline_spec.rb deleted file mode 100644 index d9906c65f344a7bb3ce9ed6b33bc97ac87237a8e..0000000000000000000000000000000000000000 --- a/ee/spec/models/vulnerabilities/finding_pipeline_spec.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Vulnerabilities::FindingPipeline, feature_category: :vulnerability_management do - describe 'associations' do - it { is_expected.to belong_to(:pipeline).class_name('Ci::Pipeline') } - it { is_expected.to belong_to(:finding).class_name('Vulnerabilities::Finding') } - end - - describe 'validations' do - let!(:finding_pipeline) { create(:vulnerabilities_finding_pipeline) } - - it { is_expected.to validate_presence_of(:finding) } - it { is_expected.to validate_presence_of(:pipeline) } - it { is_expected.to validate_uniqueness_of(:pipeline_id).scoped_to(:occurrence_id) } - end - - context 'loose foreign key on vulnerability_occurrence_pipelines.pipeline_id' do - it_behaves_like 'cleanup by a loose foreign key' do - let!(:parent) { create(:ci_pipeline) } - let!(:model) { create(:vulnerabilities_finding_pipeline, pipeline: parent) } - end - end -end diff --git a/ee/spec/models/vulnerabilities/finding_spec.rb b/ee/spec/models/vulnerabilities/finding_spec.rb index 21dd15f49e3f4f65eb5eeaca8bbf31bc17840294..0aedd312464c1c45a7ef62b13427fa16bf51f426 100644 --- a/ee/spec/models/vulnerabilities/finding_spec.rb +++ b/ee/spec/models/vulnerabilities/finding_spec.rb @@ -19,7 +19,6 @@ it { is_expected.to belong_to(:scanner).class_name('Vulnerabilities::Scanner') } it { is_expected.to belong_to(:vulnerability).inverse_of(:findings) } it { is_expected.to have_one(:one_vulnerability).class_name('Vulnerability').inverse_of(:vulnerability_finding) } - it { is_expected.to have_many(:finding_pipelines).class_name('Vulnerabilities::FindingPipeline').with_foreign_key('occurrence_id') } it { is_expected.to have_many(:identifiers).class_name('Vulnerabilities::Identifier') } it { is_expected.to have_many(:finding_identifiers).class_name('Vulnerabilities::FindingIdentifier').with_foreign_key('occurrence_id') } it { is_expected.to have_many(:finding_links).class_name('Vulnerabilities::FindingLink').with_foreign_key('vulnerability_occurrence_id') } diff --git a/ee/spec/services/vulnerabilities/removal/remove_from_project_service_spec.rb b/ee/spec/services/vulnerabilities/removal/remove_from_project_service_spec.rb index b6526b2a5c491b0af61939895e6ab8bbccee4762..d8a16af3b21ab5f99e7753241ba6d5c8ecdef0f6 100644 --- a/ee/spec/services/vulnerabilities/removal/remove_from_project_service_spec.rb +++ b/ee/spec/services/vulnerabilities/removal/remove_from_project_service_spec.rb @@ -115,7 +115,6 @@ .and not_change { Vulnerabilities::Identifier.count } .and not_change { Vulnerabilities::FindingLink.count } .and not_change { Vulnerabilities::Remediation.count } - .and not_change { Vulnerabilities::FindingPipeline.count } .and not_change { Vulnerabilities::StateTransition.count } .and not_change { Vulnerabilities::MergeRequestLink.count } .and not_change { Vulnerabilities::FindingSignature.count } diff --git a/lib/gitlab/background_migration/backfill_vulnerability_occurrence_pipelines_project_id.rb b/lib/gitlab/background_migration/backfill_vulnerability_occurrence_pipelines_project_id.rb deleted file mode 100644 index 8dfcfedc2097c4cedc1b523e56f03a4bb646ecad..0000000000000000000000000000000000000000 --- a/lib/gitlab/background_migration/backfill_vulnerability_occurrence_pipelines_project_id.rb +++ /dev/null @@ -1,10 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - class BackfillVulnerabilityOccurrencePipelinesProjectId < BackfillDesiredShardingKeyJob - operation_name :backfill_vulnerability_occurrence_pipelines_project_id - feature_category :vulnerability_management - end - end -end diff --git a/lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb b/lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb deleted file mode 100644 index e95955c450d9f5ddcabe55080811f23b8a38a3ec..0000000000000000000000000000000000000000 --- a/lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # rubocop:disable Style/Documentation - class PopulateResolvedOnDefaultBranchColumn - def perform(*); end - end - end -end - -Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn.prepend_mod_with('Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn') diff --git a/rubocop/rubocop-migrations.yml b/rubocop/rubocop-migrations.yml index c5f5adef68c91bf74c3c71df4b26cfad0a8ee804..e1e349f28df33a006bb0ea062e91a27bd52db4e1 100644 --- a/rubocop/rubocop-migrations.yml +++ b/rubocop/rubocop-migrations.yml @@ -79,7 +79,6 @@ Migration/UpdateLargeTable: - :vulnerabilities - :vulnerability_finding_links - :vulnerability_occurrence_identifiers - - :vulnerability_occurrence_pipelines - :vulnerability_occurrences - :vulnerability_reads - :web_hook_logs diff --git a/spec/db/schema_spec.rb b/spec/db/schema_spec.rb index 445efd8974a7de5984a3432a682e7728362a5b55..4a3551a7ca0ff2284beaf0ba902a6c49ab1d6940 100644 --- a/spec/db/schema_spec.rb +++ b/spec/db/schema_spec.rb @@ -205,7 +205,6 @@ user_agent_details: %w[subject_id], users: %w[color_mode_id color_scheme_id created_by_id theme_id managing_group_id], users_star_projects: %w[user_id], - vulnerability_occurrence_pipelines: %w[project_id], vulnerability_finding_links: %w[project_id], vulnerability_identifiers: %w[external_id], vulnerability_occurrence_identifiers: %w[project_id], diff --git a/spec/lib/gitlab/background_migration/backfill_vulnerability_occurrence_pipelines_project_id_spec.rb b/spec/lib/gitlab/background_migration/backfill_vulnerability_occurrence_pipelines_project_id_spec.rb deleted file mode 100644 index e79b373fca16455ec02d4a44966c02ef7d508aa4..0000000000000000000000000000000000000000 --- a/spec/lib/gitlab/background_migration/backfill_vulnerability_occurrence_pipelines_project_id_spec.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::BackfillVulnerabilityOccurrencePipelinesProjectId, - feature_category: :vulnerability_management, - schema: 20240730172815 do - include_examples 'desired sharding key backfill job' do - let(:batch_table) { :vulnerability_occurrence_pipelines } - let(:backfill_column) { :project_id } - let(:backfill_via_table) { :vulnerability_occurrences } - let(:backfill_via_column) { :project_id } - let(:backfill_via_foreign_key) { :occurrence_id } - end -end diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index 86a7f309ff8ce6c2a79e58dc01c1494f26b6b1bf..3e03d34a90131a78a1a8cbe712507dd2910be008 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -349,7 +349,6 @@ ci_pipelines: - parent_pipeline - downstream_bridges - job_artifacts -- vulnerabilities_finding_pipelines - vulnerability_findings - pipeline_config - security_scans diff --git a/spec/migrations/20240730172818_queue_backfill_vulnerability_occurrence_pipelines_project_id_spec.rb b/spec/migrations/20240730172818_queue_backfill_vulnerability_occurrence_pipelines_project_id_spec.rb index 11402da932b5bdd987bfbc94167ffe0871f7a52e..616daecaf5072031ccff3d2679ca0c3686051fbf 100644 --- a/spec/migrations/20240730172818_queue_backfill_vulnerability_occurrence_pipelines_project_id_spec.rb +++ b/spec/migrations/20240730172818_queue_backfill_vulnerability_occurrence_pipelines_project_id_spec.rb @@ -6,27 +6,14 @@ RSpec.describe QueueBackfillVulnerabilityOccurrencePipelinesProjectId, feature_category: :vulnerability_management do let!(:batched_migration) { described_class::MIGRATION } - it 'schedules a new batched migration' do + it 'does not schedule a new batched migration' do reversible_migration do |migration| migration.before -> { expect(batched_migration).not_to have_scheduled_batched_migration } migration.after -> { - expect(batched_migration).to have_scheduled_batched_migration( - table_name: :vulnerability_occurrence_pipelines, - column_name: :id, - interval: described_class::DELAY_INTERVAL, - batch_size: described_class::BATCH_SIZE, - sub_batch_size: described_class::SUB_BATCH_SIZE, - gitlab_schema: :gitlab_sec, - job_arguments: [ - :project_id, - :vulnerability_occurrences, - :project_id, - :occurrence_id - ] - ) + expect(batched_migration).not_to have_scheduled_batched_migration } end end diff --git a/spec/support/rspec_order_todo.yml b/spec/support/rspec_order_todo.yml index 04a69c382854f3d47dbc43fc3eb01e9fbd60fada..c85ae728e44b9341fe63463a98c2c263f1b94e43 100644 --- a/spec/support/rspec_order_todo.yml +++ b/spec/support/rspec_order_todo.yml @@ -1421,7 +1421,6 @@ - './ee/spec/models/vulnerabilities/finding/evidence_spec.rb' - './ee/spec/models/vulnerabilities/finding_identifier_spec.rb' - './ee/spec/models/vulnerabilities/finding_link_spec.rb' -- './ee/spec/models/vulnerabilities/finding_pipeline_spec.rb' - './ee/spec/models/vulnerabilities/finding_remediation_spec.rb' - './ee/spec/models/vulnerabilities/finding_signature_spec.rb' - './ee/spec/models/vulnerabilities/finding_spec.rb'