diff --git a/db/post_migrate/20240924095956_remove_default_for_sbom_components_organization_id.rb b/db/post_migrate/20240924095956_remove_default_for_sbom_components_organization_id.rb new file mode 100644 index 0000000000000000000000000000000000000000..983bd206c7b691983953fb071d6d34cdfd61daee --- /dev/null +++ b/db/post_migrate/20240924095956_remove_default_for_sbom_components_organization_id.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class RemoveDefaultForSbomComponentsOrganizationId < Gitlab::Database::Migration[2.2] + milestone '17.5' + + def change + change_column_default(:sbom_components, :organization_id, from: 1, to: nil) + end +end diff --git a/db/post_migrate/20240924101144_remove_default_for_sbom_component_versions_organization_id.rb b/db/post_migrate/20240924101144_remove_default_for_sbom_component_versions_organization_id.rb new file mode 100644 index 0000000000000000000000000000000000000000..b5363252c2f63786e74a1ac2471824461e6179ed --- /dev/null +++ b/db/post_migrate/20240924101144_remove_default_for_sbom_component_versions_organization_id.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class RemoveDefaultForSbomComponentVersionsOrganizationId < Gitlab::Database::Migration[2.2] + milestone '17.5' + + def change + change_column_default(:sbom_component_versions, :organization_id, from: 1, to: nil) + end +end diff --git a/db/post_migrate/20240925061340_remove_default_for_sbom_sources_organization_id.rb b/db/post_migrate/20240925061340_remove_default_for_sbom_sources_organization_id.rb new file mode 100644 index 0000000000000000000000000000000000000000..cb502c964376cca772a3606e1974acb4b1d025a0 --- /dev/null +++ b/db/post_migrate/20240925061340_remove_default_for_sbom_sources_organization_id.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class RemoveDefaultForSbomSourcesOrganizationId < Gitlab::Database::Migration[2.2] + milestone '17.5' + + def change + change_column_default(:sbom_sources, :organization_id, from: 1, to: nil) + end +end diff --git a/db/post_migrate/20240925062324_remove_default_for_sbom_source_packages_organization_id.rb b/db/post_migrate/20240925062324_remove_default_for_sbom_source_packages_organization_id.rb new file mode 100644 index 0000000000000000000000000000000000000000..5b0dfc03b56335e2a256bf8dcb45563e66c323d1 --- /dev/null +++ b/db/post_migrate/20240925062324_remove_default_for_sbom_source_packages_organization_id.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class RemoveDefaultForSbomSourcePackagesOrganizationId < Gitlab::Database::Migration[2.2] + milestone '17.5' + + def change + change_column_default(:sbom_source_packages, :organization_id, from: 1, to: nil) + end +end diff --git a/db/post_migrate/20240925063116_remove_default_for_vulnerability_exports_organization_id.rb b/db/post_migrate/20240925063116_remove_default_for_vulnerability_exports_organization_id.rb new file mode 100644 index 0000000000000000000000000000000000000000..ba773b9ca2fe5f40c8eb5979ddf51a2cc1277bf0 --- /dev/null +++ b/db/post_migrate/20240925063116_remove_default_for_vulnerability_exports_organization_id.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class RemoveDefaultForVulnerabilityExportsOrganizationId < Gitlab::Database::Migration[2.2] + milestone '17.5' + + def change + change_column_default(:vulnerability_exports, :organization_id, from: 1, to: nil) + end +end diff --git a/db/post_migrate/20240925063151_remove_default_for_vulnerability_export_parts_organization_id.rb b/db/post_migrate/20240925063151_remove_default_for_vulnerability_export_parts_organization_id.rb new file mode 100644 index 0000000000000000000000000000000000000000..e78c94295029abcd924e15f6b2be43989c15d266 --- /dev/null +++ b/db/post_migrate/20240925063151_remove_default_for_vulnerability_export_parts_organization_id.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class RemoveDefaultForVulnerabilityExportPartsOrganizationId < Gitlab::Database::Migration[2.2] + milestone '17.5' + + def change + change_column_default(:vulnerability_export_parts, :organization_id, from: 1, to: nil) + end +end diff --git a/db/post_migrate/20240925064050_remove_default_for_dependency_list_export_parts_organization_id.rb b/db/post_migrate/20240925064050_remove_default_for_dependency_list_export_parts_organization_id.rb new file mode 100644 index 0000000000000000000000000000000000000000..c3687a936f71593b17eef76bcf63a6afaa9ab5c7 --- /dev/null +++ b/db/post_migrate/20240925064050_remove_default_for_dependency_list_export_parts_organization_id.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class RemoveDefaultForDependencyListExportPartsOrganizationId < Gitlab::Database::Migration[2.2] + milestone '17.5' + + def change + change_column_default(:dependency_list_export_parts, :organization_id, from: 1, to: nil) + end +end diff --git a/db/schema_migrations/20240924095956 b/db/schema_migrations/20240924095956 new file mode 100644 index 0000000000000000000000000000000000000000..12e4ad90186978a08d10a48a8d6cf85f7c123447 --- /dev/null +++ b/db/schema_migrations/20240924095956 @@ -0,0 +1 @@ +4d9c9d1479d1ee10da51a5eb5ef86bceaff52810cfcf7c80b6e18091dd7cee71 \ No newline at end of file diff --git a/db/schema_migrations/20240924101144 b/db/schema_migrations/20240924101144 new file mode 100644 index 0000000000000000000000000000000000000000..431ea484f06f5649f48ab950612f032e1e7d00c9 --- /dev/null +++ b/db/schema_migrations/20240924101144 @@ -0,0 +1 @@ +ac57a1b0cb4399868621195eaf9b0c00ddcb63066f5927f7a90a5f11ce7b72c1 \ No newline at end of file diff --git a/db/schema_migrations/20240925061340 b/db/schema_migrations/20240925061340 new file mode 100644 index 0000000000000000000000000000000000000000..461a1a13ead2ed6f65fcba06939daebef1681d0d --- /dev/null +++ b/db/schema_migrations/20240925061340 @@ -0,0 +1 @@ +f72dc7d202643ec648fee19de40234dddfe7d1b9767b4928f64c6ffb2124837a \ No newline at end of file diff --git a/db/schema_migrations/20240925062324 b/db/schema_migrations/20240925062324 new file mode 100644 index 0000000000000000000000000000000000000000..740516775b44a4945f31f0e72bb0962d330fa6b0 --- /dev/null +++ b/db/schema_migrations/20240925062324 @@ -0,0 +1 @@ +76c2bfc3ef5ddf1fc99c02315c14da65e2306ca8544daea7687b93d3177877c3 \ No newline at end of file diff --git a/db/schema_migrations/20240925063116 b/db/schema_migrations/20240925063116 new file mode 100644 index 0000000000000000000000000000000000000000..34d401007f32683b74a8b85728628caa40fa3e12 --- /dev/null +++ b/db/schema_migrations/20240925063116 @@ -0,0 +1 @@ +48c0b97398876a4b6a2ae273a7e73432a916195ec630329d4deaa124af0d0ca8 \ No newline at end of file diff --git a/db/schema_migrations/20240925063151 b/db/schema_migrations/20240925063151 new file mode 100644 index 0000000000000000000000000000000000000000..26841a1e4a11a0b9d7a162eb7ae4f7c433abc1d0 --- /dev/null +++ b/db/schema_migrations/20240925063151 @@ -0,0 +1 @@ +e1f802ce15ed4a02d05f2b156b131cec5714ed9171e0f853717a9a3d2a6d450a \ No newline at end of file diff --git a/db/schema_migrations/20240925064050 b/db/schema_migrations/20240925064050 new file mode 100644 index 0000000000000000000000000000000000000000..fd8244ea40fd538c300911551e657ef233f34d42 --- /dev/null +++ b/db/schema_migrations/20240925064050 @@ -0,0 +1 @@ +4bf234f347198bc6598ed8ba6b3544e0071bbc9cd0d2ca9b3c4d531f94be4db5 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 2fb619b538bcfaa9ea4d158eda8f352dc8188927..2da640be842ac80b6711e7e2d9783a72d99d7b1c 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -10201,7 +10201,7 @@ CREATE TABLE dependency_list_export_parts ( dependency_list_export_id bigint NOT NULL, start_id bigint NOT NULL, end_id bigint NOT NULL, - organization_id bigint DEFAULT 1 NOT NULL, + organization_id bigint NOT NULL, created_at timestamp with time zone NOT NULL, updated_at timestamp with time zone NOT NULL, file_store integer, @@ -18005,7 +18005,7 @@ CREATE TABLE sbom_component_versions ( component_id bigint NOT NULL, version text NOT NULL, source_package_name text, - organization_id bigint DEFAULT 1 NOT NULL, + organization_id bigint NOT NULL, CONSTRAINT check_39636b9a8a CHECK ((char_length(source_package_name) <= 255)), CONSTRAINT check_e71cad08d3 CHECK ((char_length(version) <= 255)) ); @@ -18026,7 +18026,7 @@ CREATE TABLE sbom_components ( component_type smallint NOT NULL, name text NOT NULL, purl_type smallint, - organization_id bigint DEFAULT 1 NOT NULL, + organization_id bigint NOT NULL, CONSTRAINT check_91a8f6ad53 CHECK ((char_length(name) <= 255)) ); @@ -18099,7 +18099,7 @@ CREATE TABLE sbom_source_packages ( purl_type smallint NOT NULL, created_at timestamp with time zone DEFAULT now() NOT NULL, updated_at timestamp with time zone DEFAULT now() NOT NULL, - organization_id bigint DEFAULT 1 NOT NULL, + organization_id bigint NOT NULL, CONSTRAINT check_8fba79abed CHECK ((char_length(name) <= 255)) ); @@ -18118,7 +18118,7 @@ CREATE TABLE sbom_sources ( updated_at timestamp with time zone NOT NULL, source_type smallint NOT NULL, source jsonb DEFAULT '{}'::jsonb NOT NULL, - organization_id bigint DEFAULT 1 NOT NULL + organization_id bigint NOT NULL ); CREATE SEQUENCE sbom_sources_id_seq @@ -20095,7 +20095,7 @@ CREATE TABLE vulnerability_export_parts ( vulnerability_export_id bigint NOT NULL, start_id bigint NOT NULL, end_id bigint NOT NULL, - organization_id bigint DEFAULT 1 NOT NULL, + organization_id bigint NOT NULL, file_store integer, file text, created_at timestamp with time zone NOT NULL, @@ -20125,7 +20125,7 @@ CREATE TABLE vulnerability_exports ( file_store integer, format smallint DEFAULT 0 NOT NULL, group_id bigint, - organization_id bigint DEFAULT 1 NOT NULL + organization_id bigint NOT NULL ); CREATE SEQUENCE vulnerability_exports_id_seq diff --git a/ee/app/models/dependencies/dependency_list_export/part.rb b/ee/app/models/dependencies/dependency_list_export/part.rb index a7a778d04eb05d0e4a0a100e4bc109bd81aeb968..538ee3fa7825cfb810210f972b3eb6725153ba7a 100644 --- a/ee/app/models/dependencies/dependency_list_export/part.rb +++ b/ee/app/models/dependencies/dependency_list_export/part.rb @@ -4,6 +4,9 @@ module Dependencies # rubocop:disable Gitlab/BoundedContexts -- This is an exist class DependencyListExport class Part < Gitlab::Database::SecApplicationRecord include FileStoreMounter + include SafelyChangeColumnDefault + + columns_changing_default :organization_id self.table_name = 'dependency_list_export_parts' diff --git a/ee/app/models/sbom/component.rb b/ee/app/models/sbom/component.rb index 1f3ede3aa1eaee9931b9428bf6554c398e1dfcbc..d2b1fd87c2c3e3a39e5c4f7e20c70c129b6cf4b5 100644 --- a/ee/app/models/sbom/component.rb +++ b/ee/app/models/sbom/component.rb @@ -2,6 +2,10 @@ module Sbom class Component < ::Gitlab::Database::SecApplicationRecord + include SafelyChangeColumnDefault + + columns_changing_default :organization_id + has_many :occurrences, inverse_of: :component enum component_type: ::Enums::Sbom.component_types diff --git a/ee/app/models/sbom/component_version.rb b/ee/app/models/sbom/component_version.rb index 790c68b39792910eb8655f5656fd4d079ed4af22..c99a0749fe3dda3409eb0ccf043384010654a976 100644 --- a/ee/app/models/sbom/component_version.rb +++ b/ee/app/models/sbom/component_version.rb @@ -2,6 +2,10 @@ module Sbom class ComponentVersion < Gitlab::Database::SecApplicationRecord + include SafelyChangeColumnDefault + + columns_changing_default :organization_id + belongs_to :component, optional: false has_many :occurrences, inverse_of: :component_version belongs_to :organization, class_name: 'Organizations::Organization' diff --git a/ee/app/models/sbom/source.rb b/ee/app/models/sbom/source.rb index a2acbca601ee7cc5331307f38d09df93128781d9..df53a44ad5411064580c4d6f6c84134be66eef25 100644 --- a/ee/app/models/sbom/source.rb +++ b/ee/app/models/sbom/source.rb @@ -3,6 +3,9 @@ module Sbom class Source < Gitlab::Database::SecApplicationRecord include ::Sbom::SourceHelper + include SafelyChangeColumnDefault + + columns_changing_default :organization_id DEFAULT_SOURCES = { dependency_scanning: 0, diff --git a/ee/app/models/sbom/source_package.rb b/ee/app/models/sbom/source_package.rb index b4c80247166a5032e8fbabb8c65077d8cc8183c7..4587db8db583844032808b9d5b7079914bd9dcd2 100644 --- a/ee/app/models/sbom/source_package.rb +++ b/ee/app/models/sbom/source_package.rb @@ -2,6 +2,10 @@ module Sbom class SourcePackage < ::Gitlab::Database::SecApplicationRecord + include SafelyChangeColumnDefault + + columns_changing_default :organization_id + has_many :occurrences, inverse_of: :source_package enum purl_type: ::Enums::Sbom.purl_types diff --git a/ee/app/models/vulnerabilities/export.rb b/ee/app/models/vulnerabilities/export.rb index cc17c015113282e74b44362a9fc7f6aa73b0f3d1..cf3906dfdc48a88045603dd847bbcbc9494101db 100644 --- a/ee/app/models/vulnerabilities/export.rb +++ b/ee/app/models/vulnerabilities/export.rb @@ -4,6 +4,9 @@ module Vulnerabilities class Export < ApplicationRecord include Gitlab::Utils::StrongMemoize include FileStoreMounter + include SafelyChangeColumnDefault + + columns_changing_default :organization_id EXPORTER_CLASS = VulnerabilityExports::ExportService MAX_EXPORT_DURATION = 24.hours diff --git a/ee/app/models/vulnerabilities/export/part.rb b/ee/app/models/vulnerabilities/export/part.rb index 3554e5ba77acd70aabb3f0f17cc0d5c993f14a86..1d02d1855ea93dd9a31e6e9ee09d6f35b1f00229 100644 --- a/ee/app/models/vulnerabilities/export/part.rb +++ b/ee/app/models/vulnerabilities/export/part.rb @@ -4,6 +4,9 @@ module Vulnerabilities class Export class Part < Gitlab::Database::SecApplicationRecord include FileStoreMounter + include SafelyChangeColumnDefault + + columns_changing_default :organization_id self.table_name = "vulnerability_export_parts" diff --git a/lib/gitlab/background_migration/migrate_os_sbom_occurrences_to_components_without_prefix.rb b/lib/gitlab/background_migration/migrate_os_sbom_occurrences_to_components_without_prefix.rb index 32ad251a6fa584328ecfa4a0ed98b4a94eb84379..76aaa3a2e36819e7bb5cfb45c6dbcf7d8e5ec486 100644 --- a/lib/gitlab/background_migration/migrate_os_sbom_occurrences_to_components_without_prefix.rb +++ b/lib/gitlab/background_migration/migrate_os_sbom_occurrences_to_components_without_prefix.rb @@ -51,13 +51,16 @@ class Occurrence < ::ApplicationRecord belongs_to :component_version end + # rubocop:disable Metrics/AbcSize -- It went above limit when adding support for organization_id sharding key. def perform each_sub_batch do |sub_batch| # rubocop:disable Rails/FindEach -- This already operates on a sub_batch sub_batch.where("name LIKE '%/%'").each do |src_component| dst_component = Component.find_by( name: component_name_without_os_prefix(src_component.name), - purl_type: src_component.purl_type + purl_type: src_component.purl_type, + component_type: src_component.component_type, + organization_id: src_component.organization_id ) # This uses loop based batching to efficiently iterate over @@ -90,6 +93,7 @@ def perform # rubocop:enable Rails/FindEach end end + # rubocop:enable Metrics/AbcSize private @@ -104,6 +108,7 @@ def build_component_version_attributes(dst_component, occurrences) next unless occurrence.component_version.present? { component_id: dst_component.id, version: occurrence.component_version.version, + organization_id: dst_component.organization_id, source_package_name: occurrence.component_version.source_package_name } end end diff --git a/spec/lib/gitlab/background_migration/migrate_os_sbom_occurrences_to_components_without_prefix_spec.rb b/spec/lib/gitlab/background_migration/migrate_os_sbom_occurrences_to_components_without_prefix_spec.rb index e9c4b259acca482d5842acadf18752f351d3be22..4100364776c1a5fc1e9cf68446581a15298a94d0 100644 --- a/spec/lib/gitlab/background_migration/migrate_os_sbom_occurrences_to_components_without_prefix_spec.rb +++ b/spec/lib/gitlab/background_migration/migrate_os_sbom_occurrences_to_components_without_prefix_spec.rb @@ -33,22 +33,31 @@ def create_sbom_occurrence(src_component, src_component_version) end context 'when sbom occurrence belongs to sbom component with os prefix' do - let(:alpine_src_component) { sbom_components.create!(name: 'alpine/curl', purl_type: 9, component_type: 0) } + let(:alpine_src_component) do + sbom_components.create!(name: 'alpine/curl', purl_type: 9, component_type: 0, organization_id: 1) + end + let(:alpine_src_version) do sbom_component_versions.create!(version: '1.0.0', component_id: alpine_src_component.id, - source_package_name: 'curl') + source_package_name: 'curl', organization_id: 1) + end + + let(:redhat_src_component) do + sbom_components.create!(name: 'redhat/curl', purl_type: 10, component_type: 0, organization_id: 1) end - let(:redhat_src_component) { sbom_components.create!(name: 'redhat/curl', purl_type: 10, component_type: 0) } let(:redhat_src_version) do sbom_component_versions.create!(version: '1.0.0', component_id: redhat_src_component.id, - source_package_name: 'curl') + source_package_name: 'curl', organization_id: 1) + end + + let(:debian_src_component) do + sbom_components.create!(name: 'debian/curl', purl_type: 11, component_type: 0, organization_id: 1) end - let(:debian_src_component) { sbom_components.create!(name: 'debian/curl', purl_type: 11, component_type: 0) } let(:debian_src_version) do sbom_component_versions.create!(version: '1.0.0', component_id: debian_src_component.id, - source_package_name: 'curl') + source_package_name: 'curl', organization_id: 1) end subject(:perform_migration) do @@ -64,13 +73,13 @@ def create_sbom_occurrence(src_component, src_component_version) end it 'migrates the data correctly' do - alpine_dst_component = sbom_components.create!(name: 'curl', purl_type: 9, component_type: 0) + alpine_dst_component = sbom_components.create!(name: 'curl', purl_type: 9, component_type: 0, organization_id: 1) alpine_occurrence = create_sbom_occurrence(alpine_src_component, alpine_src_version) - redhat_dst_component = sbom_components.create!(name: 'curl', purl_type: 10, component_type: 0) + redhat_dst_component = sbom_components.create!(name: 'curl', purl_type: 10, component_type: 0, organization_id: 1) redhat_occurrence = create_sbom_occurrence(redhat_src_component, redhat_src_version) - debian_dst_component = sbom_components.create!(name: 'curl', purl_type: 11, component_type: 0) + debian_dst_component = sbom_components.create!(name: 'curl', purl_type: 11, component_type: 0, organization_id: 1) debian_occurrence = create_sbom_occurrence(debian_src_component, debian_src_version) perform_migration @@ -90,11 +99,13 @@ def create_sbom_occurrence(src_component, src_component_version) end context 'when components have no versions' do - let(:src_component) { sbom_components.create!(name: 'alpine/curl', purl_type: 9, component_type: 0) } + let(:src_component) do + sbom_components.create!(name: 'alpine/curl', purl_type: 9, component_type: 0, organization_id: 1) + end it 'does not raise error' do occurrence = create_sbom_occurrence(src_component, nil) - dst_component = sbom_components.create!(name: 'curl', purl_type: 9, component_type: 0) + dst_component = sbom_components.create!(name: 'curl', purl_type: 9, component_type: 0, organization_id: 1) expect { perform_migration }.not_to raise_error @@ -106,7 +117,7 @@ def create_sbom_occurrence(src_component, src_component_version) context 'when components have no occurrences' do it 'does not raise an error' do - sbom_components.create!(name: 'alpine/curl', purl_type: 9, component_type: 0) + sbom_components.create!(name: 'alpine/curl', purl_type: 9, component_type: 0, organization_id: 1) expect { perform_migration }.not_to raise_error end diff --git a/spec/lib/gitlab/background_migration/update_sbom_occurrences_component_name_based_on_pep503_spec.rb b/spec/lib/gitlab/background_migration/update_sbom_occurrences_component_name_based_on_pep503_spec.rb index 299baca54bdbe96c66e553fe64843e4162ff86ff..a536bfa157eb92eabd311dff89cb7d0c1f06804f 100644 --- a/spec/lib/gitlab/background_migration/update_sbom_occurrences_component_name_based_on_pep503_spec.rb +++ b/spec/lib/gitlab/background_migration/update_sbom_occurrences_component_name_based_on_pep503_spec.rb @@ -25,7 +25,7 @@ context 'without data' do before do - component = components.create!(name: 'azure', purl_type: 8, component_type: 0) + component = components.create!(name: 'azure', purl_type: 8, component_type: 0, organization_id: 1) occurrences.create!(project_id: project.id, component_id: component.id, commit_sha: 'commit_sha', uuid: SecureRandom.uuid, component_name: 'azure') end @@ -38,7 +38,7 @@ context 'with data' do before do %w[aws-cdk.region-info azure.identity backports.cached-property backports.csv].each do |input_name| - component = components.create!(name: input_name, purl_type: 8, component_type: 0) + component = components.create!(name: input_name, purl_type: 8, component_type: 0, organization_id: 1) occurrences.create!(project_id: project.id, component_id: component.id, commit_sha: 'commit_sha', uuid: SecureRandom.uuid, component_name: input_name) end @@ -54,7 +54,10 @@ context 'with unrelated components' do let(:component_name) { 'unrelated.component' } - let(:unrelated_component) { components.create!(name: component_name, purl_type: 6, component_type: 0) } + let(:unrelated_component) do + components.create!(name: component_name, purl_type: 6, component_type: 0, organization_id: 1) + end + let!(:unrelated_occurrence) do occurrences.create!(project_id: project.id, component_id: unrelated_component.id, commit_sha: 'commit_sha', uuid: SecureRandom.uuid, component_name: component_name)