diff --git a/app/models/packages/rpm/repository_file.rb b/app/models/packages/rpm/repository_file.rb new file mode 100644 index 0000000000000000000000000000000000000000..4b5fa59c6ee965f53b0cd98e894d2319ded83b5b --- /dev/null +++ b/app/models/packages/rpm/repository_file.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true +module Packages + module Rpm + class RepositoryFile < ApplicationRecord + include EachBatch + include UpdateProjectStatistics + include FileStoreMounter + include Packages::Installable + + INSTALLABLE_STATUSES = [:default].freeze + + enum status: { default: 0, pending_destruction: 1, processing: 2, error: 3 } + + belongs_to :project, inverse_of: :repository_files + + validates :project, presence: true + validates :file, presence: true + validates :file_name, presence: true + + mount_file_store_uploader Packages::Rpm::RepositoryFileUploader + + update_project_statistics project_statistics_name: :packages_size + end + end +end diff --git a/app/models/project.rb b/app/models/project.rb index 88cecc4770dd9bce701e9a812c81c80846df3250..aca61f645dafb9bb6590f33009a3d8b0e48e8cbb 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -236,6 +236,9 @@ def self.integration_association_name(name) # Packages has_many :packages, class_name: 'Packages::Package' has_many :package_files, through: :packages, class_name: 'Packages::PackageFile' + # repository_files must be destroyed by ruby code in order to properly remove carrierwave uploads + has_many :repository_files, inverse_of: :project, class_name: 'Packages::Rpm::RepositoryFile', + dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent # debian_distributions and associated component_files must be destroyed by ruby code in order to properly remove carrierwave uploads has_many :debian_distributions, class_name: 'Packages::Debian::ProjectDistribution', dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_one :packages_cleanup_policy, class_name: 'Packages::Cleanup::Policy', inverse_of: :project diff --git a/app/uploaders/packages/rpm/repository_file_uploader.rb b/app/uploaders/packages/rpm/repository_file_uploader.rb new file mode 100644 index 0000000000000000000000000000000000000000..ff7e2bc719a3dfa0eade484671b83fccf52b2905 --- /dev/null +++ b/app/uploaders/packages/rpm/repository_file_uploader.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true +module Packages + module Rpm + class RepositoryFileUploader < GitlabUploader + include ObjectStorage::Concern + + storage_options Gitlab.config.packages + + after :store, :schedule_background_upload + + alias_method :upload, :model + + def filename + model.file_name + end + + def store_dir + dynamic_segment + end + + private + + def dynamic_segment + raise ObjectNotReadyError, 'Repository file model not ready' unless model.id + + Gitlab::HashedPath.new( + 'projects', model.project_id, 'rpm', 'repository_files', model.id, + root_hash: model.project_id + ) + end + end + end +end diff --git a/db/docs/packages_rpm_repository_files.yml b/db/docs/packages_rpm_repository_files.yml new file mode 100644 index 0000000000000000000000000000000000000000..3aac984265cfcdf422a5cd8be0ef8bc5665fa3e0 --- /dev/null +++ b/db/docs/packages_rpm_repository_files.yml @@ -0,0 +1,9 @@ +--- +table_name: packages_rpm_repository_files +classes: +- Packages::RPM::RepositoryFile +feature_categories: +- package_registry +description: Package registry file links and file metadata for RPM packages +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/97484 +milestone: '15.5' diff --git a/db/migrate/20220912153839_create_packages_rpm_repository_file.rb b/db/migrate/20220912153839_create_packages_rpm_repository_file.rb new file mode 100644 index 0000000000000000000000000000000000000000..889720df23319519334a4249d9390605e50d2574 --- /dev/null +++ b/db/migrate/20220912153839_create_packages_rpm_repository_file.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class CreatePackagesRpmRepositoryFile < Gitlab::Database::Migration[2.0] + enable_lock_retries! + + def up + create_table :packages_rpm_repository_files do |t| + t.timestamps_with_timezone + + t.references :project, index: true, null: false, foreign_key: { on_delete: :cascade }, type: :bigint + t.integer :file_store, default: 1 + t.integer :status, default: 0, null: false, limit: 2 + t.integer :size + t.binary :file_md5 + t.binary :file_sha1 + t.binary :file_sha256 + t.text :file, null: false, limit: 255 + t.text :file_name, null: false, limit: 255 + end + end + + def down + drop_table :packages_rpm_repository_files + end +end diff --git a/db/schema_migrations/20220912153839 b/db/schema_migrations/20220912153839 new file mode 100644 index 0000000000000000000000000000000000000000..26666148feb756c9d36217c987885ed5083e8330 --- /dev/null +++ b/db/schema_migrations/20220912153839 @@ -0,0 +1 @@ +9cb59a045dd09fc956683e976d127f8f2346b2b26c25eeeadc4b0ef838fa1d02 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 122ad10e9aeed498aced27ac75ec471cd5aea27d..a7419868cd34937dd2061c31669b9603f68be167 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -18947,6 +18947,32 @@ CREATE TABLE packages_rpm_metadata ( CONSTRAINT check_c3e2fc2e89 CHECK ((char_length(release) <= 128)) ); +CREATE TABLE packages_rpm_repository_files ( + id bigint NOT NULL, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL, + project_id bigint NOT NULL, + file_store integer DEFAULT 1, + status smallint DEFAULT 0 NOT NULL, + size integer, + file_md5 bytea, + file_sha1 bytea, + file_sha256 bytea, + file text NOT NULL, + file_name text NOT NULL, + CONSTRAINT check_a9fef187f5 CHECK ((char_length(file) <= 255)), + CONSTRAINT check_b6b721b275 CHECK ((char_length(file_name) <= 255)) +); + +CREATE SEQUENCE packages_rpm_repository_files_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE packages_rpm_repository_files_id_seq OWNED BY packages_rpm_repository_files.id; + CREATE TABLE packages_rubygems_metadata ( created_at timestamp with time zone NOT NULL, updated_at timestamp with time zone NOT NULL, @@ -23862,6 +23888,8 @@ ALTER TABLE ONLY packages_package_files ALTER COLUMN id SET DEFAULT nextval('pac ALTER TABLE ONLY packages_packages ALTER COLUMN id SET DEFAULT nextval('packages_packages_id_seq'::regclass); +ALTER TABLE ONLY packages_rpm_repository_files ALTER COLUMN id SET DEFAULT nextval('packages_rpm_repository_files_id_seq'::regclass); + ALTER TABLE ONLY packages_tags ALTER COLUMN id SET DEFAULT nextval('packages_tags_id_seq'::regclass); ALTER TABLE ONLY pages_deployment_states ALTER COLUMN pages_deployment_id SET DEFAULT nextval('pages_deployment_states_pages_deployment_id_seq'::regclass); @@ -25998,6 +26026,9 @@ ALTER TABLE ONLY packages_pypi_metadata ALTER TABLE ONLY packages_rpm_metadata ADD CONSTRAINT packages_rpm_metadata_pkey PRIMARY KEY (package_id); +ALTER TABLE ONLY packages_rpm_repository_files + ADD CONSTRAINT packages_rpm_repository_files_pkey PRIMARY KEY (id); + ALTER TABLE ONLY packages_rubygems_metadata ADD CONSTRAINT packages_rubygems_metadata_pkey PRIMARY KEY (package_id); @@ -29708,6 +29739,8 @@ CREATE INDEX index_packages_project_id_name_partial_for_nuget ON packages_packag CREATE INDEX index_packages_rpm_metadata_on_package_id ON packages_rpm_metadata USING btree (package_id); +CREATE INDEX index_packages_rpm_repository_files_on_project_id ON packages_rpm_repository_files USING btree (project_id); + CREATE INDEX index_packages_tags_on_package_id ON packages_tags USING btree (package_id); CREATE INDEX index_packages_tags_on_package_id_and_updated_at ON packages_tags USING btree (package_id, updated_at DESC); @@ -34584,6 +34617,9 @@ ALTER TABLE ONLY geo_hashed_storage_attachments_events ALTER TABLE ONLY ml_candidate_params ADD CONSTRAINT fk_rails_d4a51d1185 FOREIGN KEY (candidate_id) REFERENCES ml_candidates(id); +ALTER TABLE ONLY packages_rpm_repository_files + ADD CONSTRAINT fk_rails_d545cfaed2 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; + ALTER TABLE ONLY packages_rpm_metadata ADD CONSTRAINT fk_rails_d79f02264b FOREIGN KEY (package_id) REFERENCES packages_packages(id) ON DELETE CASCADE; diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index 5c8756289530fe86fe586024705a4f59dcda8eca..e92da3fdd20af83f0957ff03f3f0440ffbfcd5a2 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -390,6 +390,7 @@ packages_nuget_dependency_link_metadata: :gitlab_main packages_nuget_metadata: :gitlab_main packages_package_file_build_infos: :gitlab_main packages_package_files: :gitlab_main +packages_rpm_repository_files: :gitlab_main packages_packages: :gitlab_main packages_pypi_metadata: :gitlab_main packages_rubygems_metadata: :gitlab_main diff --git a/spec/factories/packages/rpm/rpm_repository_files.rb b/spec/factories/packages/rpm/rpm_repository_files.rb new file mode 100644 index 0000000000000000000000000000000000000000..079d32b3995e68878c90250f2e6b678714adebc4 --- /dev/null +++ b/spec/factories/packages/rpm/rpm_repository_files.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :rpm_repository_file, class: 'Packages::Rpm::RepositoryFile' do + project + + file_name { 'repomd.xml' } + file_sha1 { 'efae869b4e95d54796a46481f3a211d6a88d0323' } + file_md5 { 'ddf8a75330c896a8d7709e75f8b5982a' } + size { 3127.kilobytes } + status { :default } + + transient do + file_metadatum_trait { :xml } + end + + transient do + file_fixture { 'spec/fixtures/packages/rpm/repodata/repomd.xml' } + end + + after(:build) do |package_file, evaluator| + package_file.file = fixture_file_upload(evaluator.file_fixture) + end + + trait(:object_storage) do + file_store { Packages::Rpm::RepositoryFileUploader::Store::REMOTE } + end + + trait :pending_destruction do + status { :pending_destruction } + end + end +end diff --git a/spec/fixtures/packages/rpm/repodata/repomd.xml b/spec/fixtures/packages/rpm/repodata/repomd.xml new file mode 100644 index 0000000000000000000000000000000000000000..4554ee9a6d0fdeef8080bc1584d6fd24fbd15777 --- /dev/null +++ b/spec/fixtures/packages/rpm/repodata/repomd.xml @@ -0,0 +1,27 @@ +<repomd xmlns="http://gitlab.com/api/v4/projects/1/packages/rpm/repodata/repomd.xml" xmlns:rpm="http://gitlab.com/api/v4/projects/1/packages/rpm/repodata/repomd.xml"> + <revision>1644602779</revision> + <data type="filelists"> + <checksum type="sha256">6503673de76312406ff8ecb06d9733c32b546a65abae4d4170d9b51fb75bf253</checksum> + <open-checksum type="sha256">7652c7496daa2507f08675a5b4f59a5428aaba72997400ae3d5e7bab8e3d9cc1</open-checksum> + <location href="repodata/6503673de76312406ff8ecb06d9733c32b546a65abae4d4170d9b51fb75bf253-filelists.xml"/> + <timestamp>1644602784</timestamp> + <size>1144067</size> + <open-size>25734004</open-size> + </data> + <data type="primary"> + <checksum type="sha256">80279a863b6236e60c3e63036b8a9a25e3764dfb3121292b91e9f583af9e7b7e</checksum> + <open-checksum type="sha256">f852f3bb39f89520434d97f6913716dc448077ad49f2e5200327367f98a89d55</open-checksum> + <location href="repodata/80279a863b6236e60c3e63036b8a9a25e3764dfb3121292b91e9f583af9e7b7e-primary.xml"/> + <timestamp>1644602784</timestamp> + <size>66996</size> + <open-size>1008586</open-size> + </data> + <data type="other"> + <checksum type="sha256">34408890500ec72c0f181542a91f7ff9320d2ef32c8e613540a5b9e1b8763e02</checksum> + <open-checksum type="sha256">acac5033036264cd26100713b014242471ade45487c28c7793466a84af512624</open-checksum> + <location href="repodata/34408890500ec72c0f181542a91f7ff9320d2ef32c8e613540a5b9e1b8763e02-other.xml"/> + <timestamp>1644602784</timestamp> + <size>43329</size> + <open-size>730393</open-size> + </data> +</repomd> diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index d8b699975ff58d933167504cd31025b40894e7fe..34c062d309642d65b704a61565b41f54cd85cf6a 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -571,6 +571,7 @@ project: - project_registry - packages - package_files +- repository_files - packages_cleanup_policy - alerting_setting - project_setting diff --git a/spec/models/factories_spec.rb b/spec/models/factories_spec.rb index 2993b2aee5856d3db2de355f1dd2e1e01ad13a25..1874a74ad962ad7a309890d97f32f9d308204890 100644 --- a/spec/models/factories_spec.rb +++ b/spec/models/factories_spec.rb @@ -25,6 +25,7 @@ def skipped_traits [:issue_customer_relations_contact, :for_contact], [:issue_customer_relations_contact, :for_issue], [:package_file, :object_storage], + [:rpm_repository_file, :object_storage], [:pages_domain, :without_certificate], [:pages_domain, :without_key], [:pages_domain, :with_missing_chain], diff --git a/spec/models/packages/rpm/repository_file_spec.rb b/spec/models/packages/rpm/repository_file_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..34347793dd83b4c0ad32e78c483fea7ebe7bffda --- /dev/null +++ b/spec/models/packages/rpm/repository_file_spec.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true +require 'spec_helper' + +RSpec.describe Packages::Rpm::RepositoryFile, type: :model do + using RSpec::Parameterized::TableSyntax + + let_it_be(:repository_file) { create(:rpm_repository_file) } + + it_behaves_like 'having unique enum values' + + describe 'relationships' do + it { is_expected.to belong_to(:project) } + end + + describe 'validations' do + it { is_expected.to validate_presence_of(:project) } + end + + context 'when updating project statistics' do + context 'when the package file has an explicit size' do + it_behaves_like 'UpdateProjectStatistics' do + subject { build(:rpm_repository_file, size: 42) } + end + end + + context 'when the package file does not have a size' do + it_behaves_like 'UpdateProjectStatistics' do + subject { build(:rpm_repository_file, size: nil) } + end + end + end + + context 'with status scopes' do + let_it_be(:pending_destruction_repository_package_file) do + create(:rpm_repository_file, :pending_destruction) + end + + describe '.with_status' do + subject { described_class.with_status(:pending_destruction) } + + it { is_expected.to contain_exactly(pending_destruction_repository_package_file) } + end + end +end diff --git a/spec/support/helpers/stub_object_storage.rb b/spec/support/helpers/stub_object_storage.rb index 1e8c69a0c0186ed6038d5dfff84425849ad0dfde..87e2a71b1cdffff175168c66668c5ff8af6bb99e 100644 --- a/spec/support/helpers/stub_object_storage.rb +++ b/spec/support/helpers/stub_object_storage.rb @@ -81,6 +81,12 @@ def stub_package_file_object_storage(**params) **params) end + def stub_rpm_repository_file_object_storage(**params) + stub_object_storage_uploader(config: Gitlab.config.packages.object_store, + uploader: ::Packages::Rpm::RepositoryFileUploader, + **params) + end + def stub_composer_cache_object_storage(**params) stub_object_storage_uploader(config: Gitlab.config.packages.object_store, uploader: ::Packages::Composer::CacheUploader, diff --git a/spec/uploaders/packages/rpm/repository_file_uploader_spec.rb b/spec/uploaders/packages/rpm/repository_file_uploader_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..720e109533b21c2bdeebdee2d260159ace41cecb --- /dev/null +++ b/spec/uploaders/packages/rpm/repository_file_uploader_spec.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true +require 'spec_helper' + +RSpec.describe Packages::Rpm::RepositoryFileUploader do + let_it_be(:repository_file) { create(:rpm_repository_file) } + let(:uploader) { described_class.new(repository_file, :file) } + let(:path) { Gitlab.config.packages.storage_path } + + subject { uploader } + + it_behaves_like 'builds correct paths', + store_dir: %r[^\h{2}/\h{2}/\h{64}/projects/\d+/rpm/repository_files/\d+$], + cache_dir: %r{/packages/tmp/cache}, + work_dir: %r{/packages/tmp/work} + + context 'when object store is remote' do + before do + stub_rpm_repository_file_object_storage + end + + include_context 'with storage', described_class::Store::REMOTE + + it_behaves_like 'builds correct paths', + store_dir: %r[^\h{2}/\h{2}/\h{64}/projects/\d+/rpm/repository_files/\d+$] + end + + describe 'remote file' do + let(:repository_file) { create(:rpm_repository_file, :object_storage) } + + context 'with object storage enabled' do + before do + stub_rpm_repository_file_object_storage + end + + it 'can store file remotely' do + allow(ObjectStorage::BackgroundMoveWorker).to receive(:perform_async) + + repository_file + + expect(repository_file.file_store).to eq(described_class::Store::REMOTE) + expect(repository_file.file.path).not_to be_blank + end + end + end +end