diff --git a/app/models/project_export_job.rb b/app/models/project_export_job.rb index c7fe3d7bc10e24f2882d9f68027181f4bb52a014..decc71ee19353182f4b32eea76d5aa2e09a203ae 100644 --- a/app/models/project_export_job.rb +++ b/app/models/project_export_job.rb @@ -2,6 +2,7 @@ class ProjectExportJob < ApplicationRecord belongs_to :project + has_many :relation_exports, class_name: 'Projects::ImportExport::RelationExport' validates :project, :jid, :status, presence: true diff --git a/app/models/projects/import_export/relation_export.rb b/app/models/projects/import_export/relation_export.rb new file mode 100644 index 0000000000000000000000000000000000000000..0a31e525ac2c54cd325c55d292e2a360a6ac1209 --- /dev/null +++ b/app/models/projects/import_export/relation_export.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Projects + module ImportExport + class RelationExport < ApplicationRecord + self.table_name = 'project_relation_exports' + + belongs_to :project_export_job + + has_one :upload, + class_name: 'Projects::ImportExport::RelationExportUpload', + foreign_key: :project_relation_export_id, + inverse_of: :relation_export + + validates :export_error, length: { maximum: 300 } + validates :jid, length: { maximum: 255 } + validates :project_export_job, presence: true + validates :relation, presence: true, length: { maximum: 255 }, uniqueness: { scope: :project_export_job_id } + validates :status, numericality: { only_integer: true }, presence: true + end + end +end diff --git a/app/models/projects/import_export/relation_export_upload.rb b/app/models/projects/import_export/relation_export_upload.rb new file mode 100644 index 0000000000000000000000000000000000000000..965dc39d19fe819859f31430a09b9d2b192c47b5 --- /dev/null +++ b/app/models/projects/import_export/relation_export_upload.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Projects + module ImportExport + class RelationExportUpload < ApplicationRecord + include WithUploads + include ObjectStorage::BackgroundMove + + self.table_name = 'project_relation_export_uploads' + + belongs_to :relation_export, + class_name: 'Projects::ImportExport::RelationExport', + foreign_key: :project_relation_export_id, + inverse_of: :upload + + mount_uploader :export_file, ImportExportUploader + end + end +end diff --git a/db/docs/project_relation_export_uploads.yml b/db/docs/project_relation_export_uploads.yml new file mode 100644 index 0000000000000000000000000000000000000000..369f6d281ee4d2ad3d40e11d23363d1d37573489 --- /dev/null +++ b/db/docs/project_relation_export_uploads.yml @@ -0,0 +1,9 @@ +--- +table_name: project_relation_export_uploads +classes: +- Projects::ImportExport::RelationExportUpload +feature_categories: +- importers +description: Used to store relation export files location +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/90624 +milestone: '15.2' diff --git a/db/docs/project_relation_exports.yml b/db/docs/project_relation_exports.yml new file mode 100644 index 0000000000000000000000000000000000000000..7014d4cae0d26523adf96af65346bf0d3e70dfc8 --- /dev/null +++ b/db/docs/project_relation_exports.yml @@ -0,0 +1,9 @@ +--- +table_name: project_relation_exports +classes: +- Projects::ImportExport::RelationExport +feature_categories: +- importers +description: Used to track the generation of relation export files for projects +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/90624 +milestone: '15.2' diff --git a/db/migrate/20220619182308_create_project_relation_exports.rb b/db/migrate/20220619182308_create_project_relation_exports.rb new file mode 100644 index 0000000000000000000000000000000000000000..7b92ca5110fbc7a1ca38f795fa842a6d009873bf --- /dev/null +++ b/db/migrate/20220619182308_create_project_relation_exports.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class CreateProjectRelationExports < Gitlab::Database::Migration[2.0] + enable_lock_retries! + + UNIQUE_INDEX_NAME = 'index_project_export_job_relation' + + def change + create_table :project_relation_exports do |t| + t.references :project_export_job, null: false, foreign_key: { on_delete: :cascade } + t.timestamps_with_timezone null: false + t.integer :status, limit: 2, null: false, default: 0 + t.text :relation, null: false, limit: 255 + t.text :jid, limit: 255 + t.text :export_error, limit: 300 + + t.index [:project_export_job_id, :relation], unique: true, name: UNIQUE_INDEX_NAME + end + end +end diff --git a/db/migrate/20220619184931_create_project_relation_export_uploads.rb b/db/migrate/20220619184931_create_project_relation_export_uploads.rb new file mode 100644 index 0000000000000000000000000000000000000000..03abf980f137ce57aa9e264854fc3befdfe3455b --- /dev/null +++ b/db/migrate/20220619184931_create_project_relation_export_uploads.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class CreateProjectRelationExportUploads < Gitlab::Database::Migration[2.0] + enable_lock_retries! + + INDEX = 'index_project_relation_export_upload_id' + + def change + create_table :project_relation_export_uploads do |t| + t.references :project_relation_export, null: false, foreign_key: { on_delete: :cascade }, index: { name: INDEX } + t.timestamps_with_timezone null: false + t.text :export_file, null: false, limit: 255 + end + end +end diff --git a/db/schema_migrations/20220619182308 b/db/schema_migrations/20220619182308 new file mode 100644 index 0000000000000000000000000000000000000000..7d85fb1c487367d26f28164adb39cb8e2719a3a9 --- /dev/null +++ b/db/schema_migrations/20220619182308 @@ -0,0 +1 @@ +f8830ecd0c49aea19857fec9b07d238f4bc269a758b6a3495d57222ab1604c74 \ No newline at end of file diff --git a/db/schema_migrations/20220619184931 b/db/schema_migrations/20220619184931 new file mode 100644 index 0000000000000000000000000000000000000000..a98c1f3e847a58e21917272e952fe67607e0b55b --- /dev/null +++ b/db/schema_migrations/20220619184931 @@ -0,0 +1 @@ +2cdbc5b29e11a2ce0679f218adc57c95d483139ca0bcd1801ea97fbd4ba68ddf \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index db8481eae64e6bc488daea940193a121c3f5c8d8..83970a070a5a2ce2afa51a77f4e618f220778257 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -19501,6 +19501,47 @@ CREATE TABLE project_pages_metadata ( onboarding_complete boolean DEFAULT false NOT NULL ); +CREATE TABLE project_relation_export_uploads ( + id bigint NOT NULL, + project_relation_export_id bigint NOT NULL, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL, + export_file text NOT NULL, + CONSTRAINT check_d8ee243e9e CHECK ((char_length(export_file) <= 255)) +); + +CREATE SEQUENCE project_relation_export_uploads_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE project_relation_export_uploads_id_seq OWNED BY project_relation_export_uploads.id; + +CREATE TABLE project_relation_exports ( + id bigint NOT NULL, + project_export_job_id bigint NOT NULL, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL, + status smallint DEFAULT 0 NOT NULL, + relation text NOT NULL, + jid text, + export_error text, + CONSTRAINT check_15e644d856 CHECK ((char_length(jid) <= 255)), + CONSTRAINT check_4b5880b795 CHECK ((char_length(relation) <= 255)), + CONSTRAINT check_dbd1cf73d0 CHECK ((char_length(export_error) <= 300)) +); + +CREATE SEQUENCE project_relation_exports_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE project_relation_exports_id_seq OWNED BY project_relation_exports.id; + CREATE TABLE project_repositories ( id bigint NOT NULL, shard_id integer NOT NULL, @@ -23316,6 +23357,10 @@ ALTER TABLE ONLY project_incident_management_settings ALTER COLUMN project_id SE ALTER TABLE ONLY project_mirror_data ALTER COLUMN id SET DEFAULT nextval('project_mirror_data_id_seq'::regclass); +ALTER TABLE ONLY project_relation_export_uploads ALTER COLUMN id SET DEFAULT nextval('project_relation_export_uploads_id_seq'::regclass); + +ALTER TABLE ONLY project_relation_exports ALTER COLUMN id SET DEFAULT nextval('project_relation_exports_id_seq'::regclass); + ALTER TABLE ONLY project_repositories ALTER COLUMN id SET DEFAULT nextval('project_repositories_id_seq'::regclass); ALTER TABLE ONLY project_repository_states ALTER COLUMN id SET DEFAULT nextval('project_repository_states_id_seq'::regclass); @@ -25454,6 +25499,12 @@ ALTER TABLE ONLY project_mirror_data ALTER TABLE ONLY project_pages_metadata ADD CONSTRAINT project_pages_metadata_pkey PRIMARY KEY (project_id); +ALTER TABLE ONLY project_relation_export_uploads + ADD CONSTRAINT project_relation_export_uploads_pkey PRIMARY KEY (id); + +ALTER TABLE ONLY project_relation_exports + ADD CONSTRAINT project_relation_exports_pkey PRIMARY KEY (id); + ALTER TABLE ONLY project_repositories ADD CONSTRAINT project_repositories_pkey PRIMARY KEY (id); @@ -29081,6 +29132,8 @@ CREATE INDEX index_project_deploy_tokens_on_deploy_token_id ON project_deploy_to CREATE UNIQUE INDEX index_project_deploy_tokens_on_project_id_and_deploy_token_id ON project_deploy_tokens USING btree (project_id, deploy_token_id); +CREATE UNIQUE INDEX index_project_export_job_relation ON project_relation_exports USING btree (project_export_job_id, relation); + CREATE UNIQUE INDEX index_project_export_jobs_on_jid ON project_export_jobs USING btree (jid); CREATE INDEX index_project_export_jobs_on_project_id_and_jid ON project_export_jobs USING btree (project_id, jid); @@ -29119,6 +29172,10 @@ CREATE INDEX index_project_pages_metadata_on_pages_deployment_id ON project_page CREATE INDEX index_project_pages_metadata_on_project_id_and_deployed_is_true ON project_pages_metadata USING btree (project_id) WHERE (deployed = true); +CREATE INDEX index_project_relation_export_upload_id ON project_relation_export_uploads USING btree (project_relation_export_id); + +CREATE INDEX index_project_relation_exports_on_project_export_job_id ON project_relation_exports USING btree (project_export_job_id); + CREATE UNIQUE INDEX index_project_repositories_on_disk_path ON project_repositories USING btree (disk_path); CREATE UNIQUE INDEX index_project_repositories_on_project_id ON project_repositories USING btree (project_id); @@ -33015,6 +33072,9 @@ ALTER TABLE ONLY design_management_versions ALTER TABLE ONLY approval_merge_request_rules_approved_approvers ADD CONSTRAINT fk_rails_6577725edb FOREIGN KEY (approval_merge_request_rule_id) REFERENCES approval_merge_request_rules(id) ON DELETE CASCADE; +ALTER TABLE ONLY project_relation_export_uploads + ADD CONSTRAINT fk_rails_660ada90c9 FOREIGN KEY (project_relation_export_id) REFERENCES project_relation_exports(id) ON DELETE CASCADE; + ALTER TABLE ONLY operations_feature_flags_clients ADD CONSTRAINT fk_rails_6650ed902c FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; @@ -33825,6 +33885,9 @@ ALTER TABLE ONLY ci_daily_build_group_report_results ALTER TABLE ONLY packages_debian_group_architectures ADD CONSTRAINT fk_rails_ef667d1b03 FOREIGN KEY (distribution_id) REFERENCES packages_debian_group_distributions(id) ON DELETE CASCADE; +ALTER TABLE ONLY project_relation_exports + ADD CONSTRAINT fk_rails_ef89b354fc FOREIGN KEY (project_export_job_id) REFERENCES project_export_jobs(id) ON DELETE CASCADE; + ALTER TABLE ONLY label_priorities ADD CONSTRAINT fk_rails_ef916d14fa FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index 6a951f266cab6c0c645e9a645a176bd6840415fc..17078dde9d86af4e17a0642715af789a1cfc5a0c 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -425,6 +425,8 @@ project_incident_management_settings: :gitlab_main project_metrics_settings: :gitlab_main project_mirror_data: :gitlab_main project_pages_metadata: :gitlab_main +project_relation_export_uploads: :gitlab_main +project_relation_exports: :gitlab_main project_repositories: :gitlab_main project_repository_states: :gitlab_main project_repository_storage_moves: :gitlab_main diff --git a/spec/factories/projects/import_export/export_relation.rb b/spec/factories/projects/import_export/export_relation.rb new file mode 100644 index 0000000000000000000000000000000000000000..2b6419dcecb9d1e4c023452a09724603365bd41b --- /dev/null +++ b/spec/factories/projects/import_export/export_relation.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :project_relation_export, class: 'Projects::ImportExport::RelationExport' do + project_export_job factory: :project_export_job + + relation { 'labels' } + status { 0 } + sequence(:jid) { |n| "project_relation_export_#{n}" } + end +end diff --git a/spec/fixtures/gitlab/import_export/labels.tar.gz b/spec/fixtures/gitlab/import_export/labels.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..8329dcf3b4a1e57fe697893180dee6a14a1185e0 Binary files /dev/null and b/spec/fixtures/gitlab/import_export/labels.tar.gz differ diff --git a/spec/models/project_export_job_spec.rb b/spec/models/project_export_job_spec.rb index 5a2b1443f8b2ab243233483687566c6c405c13c3..653d4d2df2747f09a7524341eb300dfe8ad655ab 100644 --- a/spec/models/project_export_job_spec.rb +++ b/spec/models/project_export_job_spec.rb @@ -3,17 +3,14 @@ require 'spec_helper' RSpec.describe ProjectExportJob, type: :model do - let(:project) { create(:project) } - let!(:job1) { create(:project_export_job, project: project, status: 0) } - let!(:job2) { create(:project_export_job, project: project, status: 2) } - describe 'associations' do - it { expect(job1).to belong_to(:project) } + it { is_expected.to belong_to(:project) } + it { is_expected.to have_many(:relation_exports) } end describe 'validations' do - it { expect(job1).to validate_presence_of(:project) } - it { expect(job1).to validate_presence_of(:jid) } - it { expect(job1).to validate_presence_of(:status) } + it { is_expected.to validate_presence_of(:project) } + it { is_expected.to validate_presence_of(:jid) } + it { is_expected.to validate_presence_of(:status) } end end diff --git a/spec/models/projects/import_export/relation_export_spec.rb b/spec/models/projects/import_export/relation_export_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..c74ca82e161d5996092c04c1ae39dba1a4b3b8ad --- /dev/null +++ b/spec/models/projects/import_export/relation_export_spec.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Projects::ImportExport::RelationExport, type: :model do + subject { create(:project_relation_export) } + + describe 'associations' do + it { is_expected.to belong_to(:project_export_job) } + it { is_expected.to have_one(:upload) } + end + + describe 'validations' do + it { is_expected.to validate_presence_of(:project_export_job) } + it { is_expected.to validate_presence_of(:relation) } + it { is_expected.to validate_uniqueness_of(:relation).scoped_to(:project_export_job_id) } + it { is_expected.to validate_presence_of(:status) } + it { is_expected.to validate_numericality_of(:status).only_integer } + it { is_expected.to validate_length_of(:relation).is_at_most(255) } + it { is_expected.to validate_length_of(:jid).is_at_most(255) } + it { is_expected.to validate_length_of(:export_error).is_at_most(300) } + end +end diff --git a/spec/models/projects/import_export/relation_export_upload_spec.rb b/spec/models/projects/import_export/relation_export_upload_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..c0014c5a14c574b41aee29a6a2da5baaaf58d519 --- /dev/null +++ b/spec/models/projects/import_export/relation_export_upload_spec.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Projects::ImportExport::RelationExportUpload, type: :model do + subject { described_class.new(relation_export: project_relation_export) } + + let_it_be(:project_relation_export) { create(:project_relation_export) } + + describe 'associations' do + it { is_expected.to belong_to(:relation_export) } + end + + it 'stores export file' do + stub_uploads_object_storage(ImportExportUploader, enabled: false) + + filename = 'labels.tar.gz' + subject.export_file = fixture_file_upload("spec/fixtures/gitlab/import_export/#{filename}") + + subject.save! + + url = "/uploads/-/system/projects/import_export/relation_export_upload/export_file/#{subject.id}/#{filename}" + expect(subject.export_file.url).to eq(url) + end +end