Merge branch 'rodrigo/360683-add-relation-exports-model' into 'master'

Add project export relations models to Import/Export

See merge request gitlab-org/gitlab!90624

app/models/project_export_job.rb

@@ -2,6 +2,7 @@
 
 class ProjectExportJob < ApplicationRecord
   belongs_to :project
+  has_many :relation_exports, class_name: 'Projects::ImportExport::RelationExport'
 
   validates :project, :jid, :status, presence: true
 

app/models/projects/import_export/relation_export.rb

+# frozen_string_literal: true
+
+module Projects
+  module ImportExport
+    class RelationExport < ApplicationRecord
+      self.table_name = 'project_relation_exports'
+
+      belongs_to :project_export_job
+
+      has_one :upload,
+        class_name: 'Projects::ImportExport::RelationExportUpload',
+        foreign_key: :project_relation_export_id,
+        inverse_of: :relation_export
+
+      validates :export_error, length: { maximum: 300 }
+      validates :jid, length: { maximum: 255 }
+      validates :project_export_job, presence: true
+      validates :relation, presence: true, length: { maximum: 255 }, uniqueness: { scope: :project_export_job_id }
+      validates :status, numericality: { only_integer: true }, presence: true
+    end
+  end
+end

app/models/projects/import_export/relation_export_upload.rb

+# frozen_string_literal: true
+
+module Projects
+  module ImportExport
+    class RelationExportUpload < ApplicationRecord
+      include WithUploads
+      include ObjectStorage::BackgroundMove
+
+      self.table_name = 'project_relation_export_uploads'
+
+      belongs_to :relation_export,
+        class_name: 'Projects::ImportExport::RelationExport',
+        foreign_key: :project_relation_export_id,
+        inverse_of: :upload
+
+      mount_uploader :export_file, ImportExportUploader
+    end
+  end
+end

db/docs/project_relation_export_uploads.yml

+---
+table_name: project_relation_export_uploads
+classes:
+- Projects::ImportExport::RelationExportUpload
+feature_categories:
+- importers
+description: Used to store relation export files location
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/90624
+milestone: '15.2'

db/docs/project_relation_exports.yml

+---
+table_name: project_relation_exports
+classes:
+- Projects::ImportExport::RelationExport
+feature_categories:
+- importers
+description: Used to track the generation of relation export files for projects
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/90624
+milestone: '15.2'

db/migrate/20220619182308_create_project_relation_exports.rb

+# frozen_string_literal: true
+
+class CreateProjectRelationExports < Gitlab::Database::Migration[2.0]
+  enable_lock_retries!
+
+  UNIQUE_INDEX_NAME = 'index_project_export_job_relation'
+
+  def change
+    create_table :project_relation_exports do |t|
+      t.references :project_export_job, null: false, foreign_key: { on_delete: :cascade }
+      t.timestamps_with_timezone null: false
+      t.integer :status, limit: 2, null: false, default: 0
+      t.text :relation, null: false, limit: 255
+      t.text :jid, limit: 255
+      t.text :export_error, limit: 300
+
+      t.index [:project_export_job_id, :relation], unique: true, name: UNIQUE_INDEX_NAME
+    end
+  end
+end

db/migrate/20220619184931_create_project_relation_export_uploads.rb

+# frozen_string_literal: true
+
+class CreateProjectRelationExportUploads < Gitlab::Database::Migration[2.0]
+  enable_lock_retries!
+
+  INDEX = 'index_project_relation_export_upload_id'
+
+  def change
+    create_table :project_relation_export_uploads do |t|
+      t.references :project_relation_export, null: false, foreign_key: { on_delete: :cascade }, index: { name: INDEX }
+      t.timestamps_with_timezone null: false
+      t.text :export_file, null: false, limit: 255
+    end
+  end
+end

db/schema_migrations/20220619182308

+f8830ecd0c49aea19857fec9b07d238f4bc269a758b6a3495d57222ab1604c74
\ No newline at end of file

db/schema_migrations/20220619184931

+2cdbc5b29e11a2ce0679f218adc57c95d483139ca0bcd1801ea97fbd4ba68ddf
\ No newline at end of file

db/structure.sql

@@ -19502,6 +19502,47 @@ CREATE TABLE project_pages_metadata (
     onboarding_complete boolean DEFAULT false NOT NULL
 );
+CREATE TABLE project_relation_export_uploads (
+    id bigint NOT NULL,
+    project_relation_export_id bigint NOT NULL,
+    created_at timestamp with time zone NOT NULL,
+    updated_at timestamp with time zone NOT NULL,
+    export_file text NOT NULL,
+    CONSTRAINT check_d8ee243e9e CHECK ((char_length(export_file) <= 255))
+);
+
+CREATE SEQUENCE project_relation_export_uploads_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE project_relation_export_uploads_id_seq OWNED BY project_relation_export_uploads.id;
+
+CREATE TABLE project_relation_exports (
+    id bigint NOT NULL,
+    project_export_job_id bigint NOT NULL,
+    created_at timestamp with time zone NOT NULL,
+    updated_at timestamp with time zone NOT NULL,
+    status smallint DEFAULT 0 NOT NULL,
+    relation text NOT NULL,
+    jid text,
+    export_error text,
+    CONSTRAINT check_15e644d856 CHECK ((char_length(jid) <= 255)),
+    CONSTRAINT check_4b5880b795 CHECK ((char_length(relation) <= 255)),
+    CONSTRAINT check_dbd1cf73d0 CHECK ((char_length(export_error) <= 300))
+);
+
+CREATE SEQUENCE project_relation_exports_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE project_relation_exports_id_seq OWNED BY project_relation_exports.id;
+
 CREATE TABLE project_repositories (
     id bigint NOT NULL,
     shard_id integer NOT NULL,
@@ -23317,6 +23358,10 @@ ALTER TABLE ONLY project_incident_management_settings ALTER COLUMN project_id SE
 ALTER TABLE ONLY project_mirror_data ALTER COLUMN id SET DEFAULT nextval('project_mirror_data_id_seq'::regclass);
+ALTER TABLE ONLY project_relation_export_uploads ALTER COLUMN id SET DEFAULT nextval('project_relation_export_uploads_id_seq'::regclass);
+
+ALTER TABLE ONLY project_relation_exports ALTER COLUMN id SET DEFAULT nextval('project_relation_exports_id_seq'::regclass);
+
 ALTER TABLE ONLY project_repositories ALTER COLUMN id SET DEFAULT nextval('project_repositories_id_seq'::regclass);
 ALTER TABLE ONLY project_repository_states ALTER COLUMN id SET DEFAULT nextval('project_repository_states_id_seq'::regclass);
@@ -25455,6 +25500,12 @@ ALTER TABLE ONLY project_mirror_data
 ALTER TABLE ONLY project_pages_metadata
     ADD CONSTRAINT project_pages_metadata_pkey PRIMARY KEY (project_id);
+ALTER TABLE ONLY project_relation_export_uploads
+    ADD CONSTRAINT project_relation_export_uploads_pkey PRIMARY KEY (id);
+
+ALTER TABLE ONLY project_relation_exports
+    ADD CONSTRAINT project_relation_exports_pkey PRIMARY KEY (id);
+
 ALTER TABLE ONLY project_repositories
     ADD CONSTRAINT project_repositories_pkey PRIMARY KEY (id);
@@ -29082,6 +29133,8 @@ CREATE INDEX index_project_deploy_tokens_on_deploy_token_id ON project_deploy_to
 CREATE UNIQUE INDEX index_project_deploy_tokens_on_project_id_and_deploy_token_id ON project_deploy_tokens USING btree (project_id, deploy_token_id);
+CREATE UNIQUE INDEX index_project_export_job_relation ON project_relation_exports USING btree (project_export_job_id, relation);
+
 CREATE UNIQUE INDEX index_project_export_jobs_on_jid ON project_export_jobs USING btree (jid);
 CREATE INDEX index_project_export_jobs_on_project_id_and_jid ON project_export_jobs USING btree (project_id, jid);
@@ -29120,6 +29173,10 @@ CREATE INDEX index_project_pages_metadata_on_pages_deployment_id ON project_page
 CREATE INDEX index_project_pages_metadata_on_project_id_and_deployed_is_true ON project_pages_metadata USING btree (project_id) WHERE (deployed = true);
+CREATE INDEX index_project_relation_export_upload_id ON project_relation_export_uploads USING btree (project_relation_export_id);
+
+CREATE INDEX index_project_relation_exports_on_project_export_job_id ON project_relation_exports USING btree (project_export_job_id);
+
 CREATE UNIQUE INDEX index_project_repositories_on_disk_path ON project_repositories USING btree (disk_path);
 CREATE UNIQUE INDEX index_project_repositories_on_project_id ON project_repositories USING btree (project_id);
@@ -33016,6 +33073,9 @@ ALTER TABLE ONLY design_management_versions
 ALTER TABLE ONLY approval_merge_request_rules_approved_approvers
     ADD CONSTRAINT fk_rails_6577725edb FOREIGN KEY (approval_merge_request_rule_id) REFERENCES approval_merge_request_rules(id) ON DELETE CASCADE;
+ALTER TABLE ONLY project_relation_export_uploads
+    ADD CONSTRAINT fk_rails_660ada90c9 FOREIGN KEY (project_relation_export_id) REFERENCES project_relation_exports(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY operations_feature_flags_clients
     ADD CONSTRAINT fk_rails_6650ed902c FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE;
@@ -33826,6 +33886,9 @@ ALTER TABLE ONLY ci_daily_build_group_report_results
 ALTER TABLE ONLY packages_debian_group_architectures
     ADD CONSTRAINT fk_rails_ef667d1b03 FOREIGN KEY (distribution_id) REFERENCES packages_debian_group_distributions(id) ON DELETE CASCADE;
+ALTER TABLE ONLY project_relation_exports
+    ADD CONSTRAINT fk_rails_ef89b354fc FOREIGN KEY (project_export_job_id) REFERENCES project_export_jobs(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY label_priorities
     ADD CONSTRAINT fk_rails_ef916d14fa FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE;

lib/gitlab/database/gitlab_schemas.yml

@@ -425,6 +425,8 @@ project_incident_management_settings: :gitlab_main
 project_metrics_settings: :gitlab_main
 project_mirror_data: :gitlab_main
 project_pages_metadata: :gitlab_main
+project_relation_export_uploads: :gitlab_main
+project_relation_exports: :gitlab_main
 project_repositories: :gitlab_main
 project_repository_states: :gitlab_main
 project_repository_storage_moves: :gitlab_main

spec/factories/projects/import_export/export_relation.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :project_relation_export, class: 'Projects::ImportExport::RelationExport' do
+    project_export_job factory: :project_export_job
+
+    relation { 'labels' }
+    status { 0 }
+    sequence(:jid) { |n| "project_relation_export_#{n}" }
+  end
+end

spec/models/project_export_job_spec.rb

@@ -3,17 +3,14 @@
 require 'spec_helper'
 
 RSpec.describe ProjectExportJob, type: :model do
-  let(:project) { create(:project) }
-  let!(:job1) { create(:project_export_job, project: project, status: 0) }
-  let!(:job2) { create(:project_export_job, project: project, status: 2) }
-
   describe 'associations' do
-    it { expect(job1).to belong_to(:project) }
+    it { is_expected.to belong_to(:project) }
+    it { is_expected.to have_many(:relation_exports) }
   end
 
   describe 'validations' do
-    it { expect(job1).to validate_presence_of(:project) }
-    it { expect(job1).to validate_presence_of(:jid) }
-    it { expect(job1).to validate_presence_of(:status) }
+    it { is_expected.to validate_presence_of(:project) }
+    it { is_expected.to validate_presence_of(:jid) }
+    it { is_expected.to validate_presence_of(:status) }
   end
 end

spec/models/projects/import_export/relation_export_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Projects::ImportExport::RelationExport, type: :model do
+  subject { create(:project_relation_export) }
+
+  describe 'associations' do
+    it { is_expected.to belong_to(:project_export_job) }
+    it { is_expected.to have_one(:upload) }
+  end
+
+  describe 'validations' do
+    it { is_expected.to validate_presence_of(:project_export_job) }
+    it { is_expected.to validate_presence_of(:relation) }
+    it { is_expected.to validate_uniqueness_of(:relation).scoped_to(:project_export_job_id) }
+    it { is_expected.to validate_presence_of(:status) }
+    it { is_expected.to validate_numericality_of(:status).only_integer }
+    it { is_expected.to validate_length_of(:relation).is_at_most(255) }
+    it { is_expected.to validate_length_of(:jid).is_at_most(255) }
+    it { is_expected.to validate_length_of(:export_error).is_at_most(300) }
+  end
+end

spec/models/projects/import_export/relation_export_upload_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Projects::ImportExport::RelationExportUpload, type: :model do
+  subject { described_class.new(relation_export: project_relation_export) }
+
+  let_it_be(:project_relation_export) { create(:project_relation_export) }
+
+  describe 'associations' do
+    it { is_expected.to belong_to(:relation_export) }
+  end
+
+  it 'stores export file' do
+    stub_uploads_object_storage(ImportExportUploader, enabled: false)
+
+    filename = 'labels.tar.gz'
+    subject.export_file = fixture_file_upload("spec/fixtures/gitlab/import_export/#{filename}")
+
+    subject.save!
+
+    url = "/uploads/-/system/projects/import_export/relation_export_upload/export_file/#{subject.id}/#{filename}"
+    expect(subject.export_file.url).to eq(url)
+  end
+end