diff --git a/app/models/project.rb b/app/models/project.rb
index ec77ab7011acbf9d6ffd06dfc0c7e334120351d9..a9552bbb3ffd7322325574fae0f6f7a498129f60 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -1541,6 +1541,16 @@ def github_enterprise_import?
URI.parse(import_url).host != URI.parse(Octokit::Default::API_ENDPOINT).host
end
+ # Determine whether any kind of import is in progress.
+ # - Full file import
+ # - Relation import
+ # - Direct Transfer
+ def any_import_in_progress?
+ relation_import_trackers.last&.started? ||
+ import_started? ||
+ BulkImports::Entity.with_status(:started).where(project_id: id).any?
+ end
+
def has_remote_mirror?
remote_mirror_available? && remote_mirrors.enabled.exists?
end
diff --git a/app/services/projects/import_export/relation_import_service.rb b/app/services/projects/import_export/relation_import_service.rb
new file mode 100644
index 0000000000000000000000000000000000000000..e99fb81c768b77f2dd247e638e1b990e0c460777
--- /dev/null
+++ b/app/services/projects/import_export/relation_import_service.rb
@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+# Imports a selected relation into an existing project, skipping any identified
+# duplicates. Duplicates are matched on the `iid` of the record being imported.
+module Projects
+ module ImportExport
+ class RelationImportService
+ include ::Services::ReturnServiceResponses
+
+ IMPORTABLE_RELATIONS = %w[issues merge_requests milestones ci_pipelines].freeze
+
+ # Creates a new RelationImportService.
+ #
+ # @param [User] current_user
+ # @param [Hash] params
+ # @option params [String] path The full path of the project
+ # @option params [String] relation The relation to import. See IMPORTABLE_RELATIONS for permitted values.
+ # @option params [UploadedFile] file The export archive containing the data to import
+ def initialize(current_user:, params:)
+ @current_user = current_user
+ @params = params
+ end
+
+ # Checks the validity of the chosen project and triggers the re-import of
+ # the chosen relation.
+ #
+ # @return [Services::ServiceResponse]
+ def execute
+ return error(_('Project not found'), :not_found) unless project
+
+ unless relation_valid?
+ return error(
+ format(
+ _('Imported relation must be one of %{relations}'),
+ relations: IMPORTABLE_RELATIONS.to_sentence(last_word_connector: ', or ')
+ ),
+ :bad_request
+ )
+ end
+
+ return error(_('You are not authorized to perform this action'), :forbidden) unless user_permitted?
+ return error(_('A relation import is already in progress for this project'), :conflict) if import_in_progress?
+
+ tracker = create_status_tracker
+
+ attach_import_file
+
+ Projects::ImportExport::RelationImportWorker.perform_async(
+ tracker.id,
+ current_user.id
+ )
+
+ success(tracker)
+ end
+
+ private
+
+ attr_reader :current_user, :params
+
+ def user_permitted?
+ Ability.allowed?(current_user, :admin_project, project)
+ end
+
+ def relation_valid?
+ IMPORTABLE_RELATIONS.include?(params[:relation])
+ end
+
+ def attach_import_file
+ project.update(import_export_upload: ImportExportUpload.new(import_file: params[:file]))
+ end
+
+ def create_status_tracker
+ project.relation_import_trackers.create(
+ relation: params[:relation]
+ )
+ end
+
+ def project
+ @project ||= Project.find_by_full_path(params[:path])
+ end
+
+ def import_in_progress?
+ project.any_import_in_progress?
+ end
+ end
+ end
+end
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index 5ec4a96802513ec1363291761345012ed5e869fb..45d74c93656d21d338aa62de54b297e4c2860be1 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -3765,6 +3765,15 @@
:weight: 1
:idempotent: true
:tags: []
+- :name: projects_import_export_relation_import
+ :worker_name: Projects::ImportExport::RelationImportWorker
+ :feature_category: :importers
+ :has_external_dependencies: false
+ :urgency: :low
+ :resource_boundary: :memory
+ :weight: 1
+ :idempotent: true
+ :tags: []
- :name: projects_import_export_wait_relation_exports
:worker_name: Projects::ImportExport::WaitRelationExportsWorker
:feature_category: :importers
diff --git a/app/workers/projects/import_export/relation_import_worker.rb b/app/workers/projects/import_export/relation_import_worker.rb
new file mode 100644
index 0000000000000000000000000000000000000000..c30b20593c49dd57e65006a565cff1a7977dab97
--- /dev/null
+++ b/app/workers/projects/import_export/relation_import_worker.rb
@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module Projects
+ module ImportExport
+ class RelationImportWorker
+ include ApplicationWorker
+
+ sidekiq_options retry: 6
+
+ idempotent!
+ data_consistency :delayed
+ deduplicate :until_executed
+ feature_category :importers
+ urgency :low
+ worker_resource_boundary :memory
+
+ attr_reader :tracker, :project, :current_user
+
+ def perform(tracker_id, user_id)
+ @current_user = User.find(user_id)
+ @tracker = ::Projects::ImportExport::RelationImportTracker.find(tracker_id)
+ @project = tracker.project
+
+ return unless tracker.can_start?
+
+ tracker.start!
+
+ extract_import_file
+ process_import
+
+ tracker.finish!
+ rescue StandardError => error
+ failure_service = Gitlab::ImportExport::ImportFailureService.new(project)
+ failure_service.log_import_failure(
+ source: 'RelationImportWorker#perform',
+ exception: error,
+ relation_key: tracker.relation
+ )
+
+ tracker.fail_op!
+
+ raise
+ end
+
+ private
+
+ def extract_import_file
+ Gitlab::ImportExport::FileImporter.import(
+ importable: project,
+ archive_file: project.import_export_upload.import_file.path,
+ shared: shared_export_data
+ )
+ end
+
+ def process_import
+ tree_restorer = Gitlab::ImportExport::Project::RelationTreeRestorer.new(
+ user: current_user,
+ shared: shared_export_data,
+ relation_reader: relation_reader,
+ object_builder: Gitlab::ImportExport::Project::ObjectBuilder,
+ members_mapper: members_mapper,
+ relation_factory: Gitlab::ImportExport::Project::RelationFactory,
+ reader: Gitlab::ImportExport::Reader.new(shared: shared_export_data),
+ importable: project,
+ importable_attributes: relation_reader.consume_attributes('project'),
+ importable_path: 'project',
+ skip_on_duplicate_iid: true
+ )
+
+ tree_restorer.restore_single_relation(tracker.relation)
+ end
+
+ def relation_reader
+ @relation_reader ||= Gitlab::ImportExport::Json::NdjsonReader.new(
+ File.join(shared_export_data.export_path, 'tree')
+ )
+ end
+
+ def members_mapper
+ project_members = relation_reader
+ .consume_relation('project', 'project_members', mark_as_consumed: false)
+ .map(&:first)
+
+ Gitlab::ImportExport::MembersMapper.new(
+ exported_members: project_members,
+ user: current_user,
+ importable: project
+ )
+ end
+
+ def shared_export_data
+ @shared ||= project.import_export_shared
+ end
+ end
+ end
+end
diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml
index 2233e6ba322814b5306b566299b758271395b716..ab4b7d200ac3f0f81f494d7f471acc5ddc90172d 100644
--- a/config/sidekiq_queues.yml
+++ b/config/sidekiq_queues.yml
@@ -611,6 +611,8 @@
- 1
- - projects_import_export_relation_export
- 1
+- - projects_import_export_relation_import
+ - 1
- - projects_import_export_wait_relation_exports
- 1
- - projects_inactive_projects_deletion_notification
diff --git a/lib/gitlab/import_export/group/relation_tree_restorer.rb b/lib/gitlab/import_export/group/relation_tree_restorer.rb
index ae47c95036ea5003df00d9ea01eb72661af05074..f800bae2548d58ad0e187cf2f1524ff9e10d75af 100644
--- a/lib/gitlab/import_export/group/relation_tree_restorer.rb
+++ b/lib/gitlab/import_export/group/relation_tree_restorer.rb
@@ -4,6 +4,8 @@ module Gitlab
module ImportExport
module Group
class RelationTreeRestorer
+ include Gitlab::Utils::StrongMemoize
+
def initialize( # rubocop:disable Metrics/ParameterLists
user:,
shared:,
@@ -14,7 +16,8 @@ def initialize( # rubocop:disable Metrics/ParameterLists
reader:,
importable:,
importable_attributes:,
- importable_path:
+ importable_path:,
+ skip_on_duplicate_iid: false
)
@user = user
@shared = shared
@@ -26,15 +29,28 @@ def initialize( # rubocop:disable Metrics/ParameterLists
@reader = reader
@importable_attributes = importable_attributes
@importable_path = importable_path
+ @skip_on_duplicate_iid = skip_on_duplicate_iid
end
def restore
+ bulk_insert_without_cache_or_touch do
+ create_relations!
+ end
+ end
+
+ def restore_single_relation(relation_key)
+ # NO-OP. This is currently only available for file-based project import.
+ end
+
+ private
+
+ def bulk_insert_without_cache_or_touch
Gitlab::Database.all_uncached do
ActiveRecord::Base.no_touching do
update_params!
BulkInsertableAssociations.with_bulk_insert(enabled: bulk_insert_enabled) do
- create_relations!
+ yield
end
end
end
@@ -48,12 +64,14 @@ def restore
false
end
- private
-
def bulk_insert_enabled
false
end
+ def skip_on_duplicate_iid?
+ @skip_on_duplicate_iid
+ end
+
# Loops through the tree of models defined in import_export.yml and
# finds them in the imported JSON so they can be instantiated and saved
# in the DB. The structure and relationships between models are guessed from
@@ -73,8 +91,10 @@ def process_relation!(relation_key, relation_definition)
def process_relation_item!(relation_key, relation_definition, relation_index, data_hash)
relation_object = build_relation(relation_key, relation_definition, relation_index, data_hash)
+
return unless relation_object
return if relation_invalid_for_importable?(relation_object)
+ return if skip_on_duplicate_iid? && previously_imported?(relation_object, relation_key)
relation_object.assign_attributes(importable_class_sym => @importable)
@@ -88,6 +108,25 @@ def process_relation_item!(relation_key, relation_definition, relation_index, da
external_identifiers: external_identifiers(data_hash))
end
+ def previously_imported?(relation_object, relation_key)
+ existing_iids(relation_key).key?(relation_object.iid)
+ end
+
+ # Generate the list of existing IIDs as a hash.
+ # { issues: { 1: true, 2: true, ... }}
+ # A hash is used rather than returning the array of IDs because lookup
+ # performance is greatly improved.
+ def existing_iids(relation_key)
+ strong_memoize_with(:existing_iids, relation_key) do
+ case relation_key
+ when 'issues' then @importable.issues.pluck(:iid)
+ when 'milestones' then @importable.milestones.pluck(:iid)
+ when 'ci_pipelines' then @importable.ci_pipelines.pluck(:iid)
+ when 'merge_requests' then @importable.merge_requests.pluck(:iid)
+ end.index_with(true)
+ end
+ end
+
def save_relation_object(relation_object, relation_key, relation_definition, relation_index)
if relation_object.new_record?
saver = Gitlab::ImportExport::Base::RelationObjectSaver.new(
diff --git a/lib/gitlab/import_export/project/relation_tree_restorer.rb b/lib/gitlab/import_export/project/relation_tree_restorer.rb
index b5247754199e88a14ffb8b68a6780009c49e35b4..056c30a996f9d6997f342551fff08e3dd728774d 100644
--- a/lib/gitlab/import_export/project/relation_tree_restorer.rb
+++ b/lib/gitlab/import_export/project/relation_tree_restorer.rb
@@ -7,6 +7,12 @@ class RelationTreeRestorer < ImportExport::Group::RelationTreeRestorer
# Relations which cannot be saved at project level (and have a group assigned)
GROUP_MODELS = [GroupLabel, Milestone, Epic].freeze
+ def restore_single_relation(relation_key)
+ bulk_insert_without_cache_or_touch do
+ process_relation!(relation_key, relations[relation_key])
+ end
+ end
+
private
def group_models
diff --git a/locale/gitlab.pot b/locale/gitlab.pot
index da277161f23464234a87b1b81e9e020ef102c941..6f63b2052273ac6830c321e86374386d51c92426 100644
--- a/locale/gitlab.pot
+++ b/locale/gitlab.pot
@@ -1957,6 +1957,9 @@ msgstr ""
msgid "A ready-to-go template for use with iOS Swift apps"
msgstr ""
+msgid "A relation import is already in progress for this project"
+msgstr ""
+
msgid "A release with a date in the future is labeled as an %{linkStart}Upcoming Release%{linkEnd}."
msgstr ""
@@ -26355,6 +26358,9 @@ msgstr ""
msgid "ImportProjects|Update of imported projects with realtime changes failed"
msgstr ""
+msgid "Imported relation must be one of %{relations}"
+msgstr ""
+
msgid "Imported requirements"
msgstr ""
@@ -39525,6 +39531,9 @@ msgstr ""
msgid "Project navigation"
msgstr ""
+msgid "Project not found"
+msgstr ""
+
msgid "Project or Group"
msgstr ""
diff --git a/spec/commands/sidekiq_cluster/cli_spec.rb b/spec/commands/sidekiq_cluster/cli_spec.rb
index f6511007a5111e247110a7ee0cbe2c47d5a4180a..1d8196a6a19a40271f494c4bee8230fd4e309eac 100644
--- a/spec/commands/sidekiq_cluster/cli_spec.rb
+++ b/spec/commands/sidekiq_cluster/cli_spec.rb
@@ -262,12 +262,12 @@
if Gitlab.ee?
[
%w[incident_management_close_incident status_page_publish],
- %w[bulk_imports_pipeline bulk_imports_pipeline_batch bulk_imports_relation_batch_export bulk_imports_relation_export project_export projects_import_export_parallel_project_export projects_import_export_relation_export repository_import project_template_export]
+ %w[bulk_imports_pipeline bulk_imports_pipeline_batch bulk_imports_relation_batch_export bulk_imports_relation_export project_export projects_import_export_parallel_project_export projects_import_export_relation_export projects_import_export_relation_import repository_import project_template_export]
]
else
[
%w[incident_management_close_incident],
- %w[bulk_imports_pipeline bulk_imports_pipeline_batch bulk_imports_relation_batch_export bulk_imports_relation_export project_export projects_import_export_parallel_project_export projects_import_export_relation_export repository_import]
+ %w[bulk_imports_pipeline bulk_imports_pipeline_batch bulk_imports_relation_batch_export bulk_imports_relation_export project_export projects_import_export_parallel_project_export projects_import_export_relation_export projects_import_export_relation_import repository_import]
]
end
diff --git a/spec/factories/projects/import_export/relation_import_tracker.rb b/spec/factories/projects/import_export/relation_import_tracker.rb
index 255b7d7c04ba34f64de73010abf27d3fd0809db9..5dc69242e71b32f05e5683564d0653c4dbb7274d 100644
--- a/spec/factories/projects/import_export/relation_import_tracker.rb
+++ b/spec/factories/projects/import_export/relation_import_tracker.rb
@@ -6,5 +6,9 @@
relation { :issues }
status { 0 }
+
+ trait :started do
+ status { 1 }
+ end
end
end
diff --git a/spec/lib/gitlab/import_export/group/relation_tree_restorer_spec.rb b/spec/lib/gitlab/import_export/group/relation_tree_restorer_spec.rb
index 8ab99875a0a8b7bd8d1ad1dd1ff049b693124651..c31451ac83343105bc06f587830cf0eae2488508 100644
--- a/spec/lib/gitlab/import_export/group/relation_tree_restorer_spec.rb
+++ b/spec/lib/gitlab/import_export/group/relation_tree_restorer_spec.rb
@@ -10,164 +10,166 @@
require 'spec_helper'
RSpec.describe Gitlab::ImportExport::Group::RelationTreeRestorer, feature_category: :importers do
- let(:group) { create(:group).tap { |g| g.add_owner(user) } }
- let(:importable) { create(:group, parent: group) }
+ describe '#restore' do
+ let(:group) { create(:group).tap { |g| g.add_owner(user) } }
+ let(:importable) { create(:group, parent: group) }
- include_context 'relation tree restorer shared context' do
- let(:importable_name) { 'groups/4353' }
- end
-
- let(:path) { Rails.root.join('spec/fixtures/lib/gitlab/import_export/group_exports/no_children/tree') }
- let(:relation_reader) do
- Gitlab::ImportExport::Json::NdjsonReader.new(path)
- end
+ include_context 'relation tree restorer shared context' do
+ let(:importable_name) { 'groups/4353' }
+ end
- let(:reader) do
- Gitlab::ImportExport::Reader.new(
- shared: shared,
- config: Gitlab::ImportExport::Config.new(config: Gitlab::ImportExport.group_config_file).to_h
- )
- end
+ let(:path) { Rails.root.join('spec/fixtures/lib/gitlab/import_export/group_exports/no_children/tree') }
+ let(:relation_reader) do
+ Gitlab::ImportExport::Json::NdjsonReader.new(path)
+ end
- let(:members_mapper) do
- Gitlab::ImportExport::MembersMapper.new(
- exported_members: relation_reader.consume_relation(importable_name, 'members').map(&:first),
- user: user,
- importable: importable
- )
- end
+ let(:reader) do
+ Gitlab::ImportExport::Reader.new(
+ shared: shared,
+ config: Gitlab::ImportExport::Config.new(config: Gitlab::ImportExport.group_config_file).to_h
+ )
+ end
- let(:relation_tree_restorer) do
- described_class.new(
- user: user,
- shared: shared,
- relation_reader: relation_reader,
- object_builder: Gitlab::ImportExport::Group::ObjectBuilder,
- members_mapper: members_mapper,
- relation_factory: Gitlab::ImportExport::Group::RelationFactory,
- reader: reader,
- importable: importable,
- importable_path: importable_name,
- importable_attributes: attributes
- )
- end
+ let(:members_mapper) do
+ Gitlab::ImportExport::MembersMapper.new(
+ exported_members: relation_reader.consume_relation(importable_name, 'members').map(&:first),
+ user: user,
+ importable: importable
+ )
+ end
- subject(:restore_relations) { relation_tree_restorer.restore }
+ let(:relation_tree_restorer) do
+ described_class.new(
+ user: user,
+ shared: shared,
+ relation_reader: relation_reader,
+ object_builder: Gitlab::ImportExport::Group::ObjectBuilder,
+ members_mapper: members_mapper,
+ relation_factory: Gitlab::ImportExport::Group::RelationFactory,
+ reader: reader,
+ importable: importable,
+ importable_path: importable_name,
+ importable_attributes: attributes
+ )
+ end
- it 'restores group tree' do
- expect(restore_relations).to eq(true)
- end
+ subject(:restore_relations) { relation_tree_restorer.restore }
- it 'logs top-level relation creation' do
- expect(shared.logger)
- .to receive(:info)
- .with(hash_including(message: '[Project/Group Import] Created new object relation'))
- .at_least(:once)
+ it 'restores group tree' do
+ expect(restore_relations).to eq(true)
+ end
- restore_relations
- end
+ it 'logs top-level relation creation' do
+ expect(shared.logger)
+ .to receive(:info)
+ .with(hash_including(message: '[Project/Group Import] Created new object relation'))
+ .at_least(:once)
- describe 'relation object saving' do
- before do
- allow(shared.logger).to receive(:info).and_call_original
- allow(relation_reader).to receive(:consume_relation).and_call_original
+ restore_relations
end
- context 'when relation object is new' do
+ describe 'relation object saving' do
before do
- allow(relation_reader)
- .to receive(:consume_relation)
- .with(importable_name, 'boards')
- .and_return([[board, 0]])
+ allow(shared.logger).to receive(:info).and_call_original
+ allow(relation_reader).to receive(:consume_relation).and_call_original
end
- context 'when relation object has invalid subrelations' do
- let(:board) do
- {
- 'name' => 'test',
- 'lists' => [List.new, List.new],
- 'group_id' => importable.id
- }
+ context 'when relation object is new' do
+ before do
+ allow(relation_reader)
+ .to receive(:consume_relation)
+ .with(importable_name, 'boards')
+ .and_return([[board, 0]])
end
- it 'logs invalid subrelations' do
- expect(shared.logger)
- .to receive(:info)
- .with(
- message: '[Project/Group Import] Invalid subrelation',
- group_id: importable.id,
- relation_key: 'boards',
- error_messages: "Label can't be blank, Position can't be blank, and Position is not a number"
- )
-
- restore_relations
-
- board = importable.boards.last
- failure = importable.import_failures.first
-
- expect(importable.import_failures.count).to eq(2)
- expect(board.name).to eq('test')
- expect(failure.exception_class).to eq('ActiveRecord::RecordInvalid')
- expect(failure.source).to eq('RelationTreeRestorer#save_relation_object')
- expect(failure.exception_message)
- .to eq("Label can't be blank, Position can't be blank, and Position is not a number")
+ context 'when relation object has invalid subrelations' do
+ let(:board) do
+ {
+ 'name' => 'test',
+ 'lists' => [List.new, List.new],
+ 'group_id' => importable.id
+ }
+ end
+
+ it 'logs invalid subrelations' do
+ expect(shared.logger)
+ .to receive(:info)
+ .with(
+ message: '[Project/Group Import] Invalid subrelation',
+ group_id: importable.id,
+ relation_key: 'boards',
+ error_messages: "Label can't be blank, Position can't be blank, and Position is not a number"
+ )
+
+ restore_relations
+
+ board = importable.boards.last
+ failure = importable.import_failures.first
+
+ expect(importable.import_failures.count).to eq(2)
+ expect(board.name).to eq('test')
+ expect(failure.exception_class).to eq('ActiveRecord::RecordInvalid')
+ expect(failure.source).to eq('RelationTreeRestorer#save_relation_object')
+ expect(failure.exception_message)
+ .to eq("Label can't be blank, Position can't be blank, and Position is not a number")
+ end
end
end
- end
- context 'when invalid relation object has a loggable external identifier' do
- before do
- allow(relation_reader)
- .to receive(:consume_relation)
- .with(importable_name, 'milestones')
- .and_return([
- [invalid_milestone, 0],
- [invalid_milestone_with_no_iid, 1]
- ])
- end
+ context 'when invalid relation object has a loggable external identifier' do
+ before do
+ allow(relation_reader)
+ .to receive(:consume_relation)
+ .with(importable_name, 'milestones')
+ .and_return([
+ [invalid_milestone, 0],
+ [invalid_milestone_with_no_iid, 1]
+ ])
+ end
- let(:invalid_milestone) { build(:milestone, iid: 123, name: nil) }
- let(:invalid_milestone_with_no_iid) { build(:milestone, iid: nil, name: nil) }
+ let(:invalid_milestone) { build(:milestone, iid: 123, name: nil) }
+ let(:invalid_milestone_with_no_iid) { build(:milestone, iid: nil, name: nil) }
- it 'logs invalid record with external identifier' do
- restore_relations
+ it 'logs invalid record with external identifier' do
+ restore_relations
- iids_for_failures = importable.import_failures.collect { |f| [f.relation_key, f.external_identifiers] }
- expected_iids = [
- ["milestones", { "iid" => invalid_milestone.iid }],
- ["milestones", {}]
- ]
+ iids_for_failures = importable.import_failures.collect { |f| [f.relation_key, f.external_identifiers] }
+ expected_iids = [
+ ["milestones", { "iid" => invalid_milestone.iid }],
+ ["milestones", {}]
+ ]
- expect(iids_for_failures).to match_array(expected_iids)
+ expect(iids_for_failures).to match_array(expected_iids)
+ end
end
- end
- context 'when relation object is persisted' do
- before do
- allow(relation_reader)
- .to receive(:consume_relation)
- .with(importable_name, 'labels')
- .and_return([[label, 0]])
- end
+ context 'when relation object is persisted' do
+ before do
+ allow(relation_reader)
+ .to receive(:consume_relation)
+ .with(importable_name, 'labels')
+ .and_return([[label, 0]])
+ end
- context 'when relation object is invalid' do
- let(:label) { create(:group_label, group: group, title: 'test') }
+ context 'when relation object is invalid' do
+ let(:label) { create(:group_label, group: group, title: 'test') }
- it 'saves import failure with nested errors' do
- label.priorities << [LabelPriority.new, LabelPriority.new]
+ it 'saves import failure with nested errors' do
+ label.priorities << [LabelPriority.new, LabelPriority.new]
- restore_relations
+ restore_relations
- failure = importable.import_failures.first
+ failure = importable.import_failures.first
- expect(importable.labels.count).to eq(0)
- expect(importable.import_failures.count).to eq(1)
- expect(failure.exception_class).to eq('ActiveRecord::RecordInvalid')
- expect(failure.source).to eq('process_relation_item!')
- expect(failure.exception_message)
- .to eq("Validation failed: Priorities is invalid, Project can't be blank, Priority can't be blank, " \
- "Priority is not a number, Project can't be blank, Priority can't be blank, " \
- "Priority is not a number")
+ expect(importable.labels.count).to eq(0)
+ expect(importable.import_failures.count).to eq(1)
+ expect(failure.exception_class).to eq('ActiveRecord::RecordInvalid')
+ expect(failure.source).to eq('process_relation_item!')
+ expect(failure.exception_message)
+ .to eq("Validation failed: Priorities is invalid, Project can't be blank, Priority can't be blank, " \
+ "Priority is not a number, Project can't be blank, Priority can't be blank, " \
+ "Priority is not a number")
+ end
end
end
end
diff --git a/spec/lib/gitlab/import_export/project/relation_tree_restorer_spec.rb b/spec/lib/gitlab/import_export/project/relation_tree_restorer_spec.rb
index 0f4f2eb573c03e6426ef5bca27e8cbb660ced593..f610d48ca90d5631150017c2ac4f3588ac209fa7 100644
--- a/spec/lib/gitlab/import_export/project/relation_tree_restorer_spec.rb
+++ b/spec/lib/gitlab/import_export/project/relation_tree_restorer_spec.rb
@@ -88,4 +88,134 @@
relation_tree_restorer.restore
end
end
+
+ describe '#restore_single_relation' do
+ let_it_be(:importable) { create(:project) }
+
+ let(:relation_reader) do
+ Gitlab::ImportExport::Json::NdjsonReader.new(
+ 'spec/fixtures/lib/gitlab/import_export/complex/tree'
+ )
+ end
+
+ let(:relation_tree_restorer) do
+ described_class.new(
+ user: user,
+ shared: shared,
+ relation_reader: relation_reader,
+ object_builder: Gitlab::ImportExport::Project::ObjectBuilder,
+ members_mapper: members_mapper,
+ relation_factory: Gitlab::ImportExport::Project::RelationFactory,
+ reader: reader,
+ importable: importable,
+ importable_path: importable_name,
+ importable_attributes: attributes,
+ skip_on_duplicate_iid: skip_on_duplicate_iid
+ )
+ end
+
+ subject(:restore_relations) { relation_tree_restorer.restore_single_relation(relation_key) }
+
+ shared_examples 'saving single relation' do
+ context 'when skipping existing IIDs' do
+ let(:skip_on_duplicate_iid) { true }
+
+ it 'does not attempt to save the duplicate relation' do
+ expect(relation_tree_restorer).not_to receive(:save_relation_object)
+
+ restore_relations
+ end
+ end
+
+ context 'when not skipping existing IIDs' do
+ let(:skip_on_duplicate_iid) { false }
+
+ it 'attempts to save the duplicate relation' do
+ expect(relation_tree_restorer).to receive(:save_relation_object).once
+
+ restore_relations
+ end
+ end
+ end
+
+ context 'when importing issues' do
+ let(:relation_key) { 'issues' }
+
+ before do
+ importable.issues.create!(iid: 123, title: 'Issue', author: user)
+
+ allow(relation_reader)
+ .to receive(:consume_relation)
+ .with(importable_name, 'issues')
+ .and_return([[build(:issue, iid: 123, title: 'Issue', author_id: user.id), 0]])
+ end
+
+ include_examples 'saving single relation'
+ end
+
+ context 'when importing milestones' do
+ let(:relation_key) { 'milestones' }
+
+ before do
+ importable.milestones.create!(iid: 123, title: 'Milestone')
+
+ allow(relation_reader)
+ .to receive(:consume_relation)
+ .with(importable_name, 'milestones')
+ .and_return([[build(:milestone, iid: 123, name: 'Milestone'), 0]])
+ end
+
+ include_examples 'saving single relation'
+ end
+
+ context 'when importing CI pipelines' do
+ let(:relation_key) { 'ci_pipelines' }
+
+ before do
+ create(
+ :ci_pipeline,
+ project: importable,
+ iid: 123
+ )
+
+ allow(relation_reader)
+ .to receive(:consume_relation)
+ .with(importable_name, 'ci_pipelines')
+ .and_return([[build(:ci_pipeline, iid: 123), 0]])
+ end
+
+ include_examples 'saving single relation'
+ end
+
+ context 'when importing merge requests' do
+ let(:relation_key) { 'merge_requests' }
+
+ before do
+ create(
+ :merge_request,
+ iid: 123,
+ source_project: importable,
+ target_project: importable
+ )
+
+ allow(relation_reader)
+ .to receive(:consume_relation)
+ .with(importable_name, 'merge_requests')
+ .and_return([[build(:merge_request, iid: 123), 0]])
+ end
+
+ include_examples 'saving single relation'
+ end
+
+ context 'when importing an unknown relation' do
+ let(:relation_key) { 'unknown' }
+ let(:skip_on_duplicate_iid) { false }
+
+ it 'does not attempt an import' do
+ expect(relation_tree_restorer).not_to receive(:save_relation_object)
+
+ restore_relations
+ end
+ end
+ end
end
diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb
index 5a3d5c147e5e47bcf94f030fd833da6d94ae2bb5..e1427ce3a902329216a0e27af8889353834f5065 100644
--- a/spec/models/project_spec.rb
+++ b/spec/models/project_spec.rb
@@ -4032,6 +4032,40 @@ def has_external_wiki
it { expect(project.gitea_import?).to be true }
end
+ describe '#any_import_in_progress?' do
+ let_it_be_with_reload(:project) { create(:project) }
+
+ subject { project.any_import_in_progress? }
+
+ context 'when a file import is in progress' do
+ before do
+ create(:import_state, :started, project: project)
+ end
+
+ it { is_expected.to be_truthy }
+ end
+
+ context 'when a relation import is in progress' do
+ before do
+ create(:relation_import_tracker, :started, project: project)
+ end
+
+ it { is_expected.to be_truthy }
+ end
+
+ context 'when direct transfer is in progress' do
+ before do
+ create(:bulk_import_entity, :project_entity, :started, project: project)
+ end
+
+ it { is_expected.to be_truthy }
+ end
+
+ context 'when no imports are in progress' do
+ it { is_expected.to be_falsy }
+ end
+ end
+
describe '#has_remote_mirror?' do
let(:project) { create(:project, :remote_mirror, :import_started) }
diff --git a/spec/services/projects/import_export/relation_import_service_spec.rb b/spec/services/projects/import_export/relation_import_service_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..6c2dc49d4605ddd280a18d8116e0344d74c4fb59
--- /dev/null
+++ b/spec/services/projects/import_export/relation_import_service_spec.rb
@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.shared_examples 'returns failure response' do |expected_status|
+ it 'returns an error status' do
+ response = import_service.execute
+
+ expect(response).to be_instance_of(ServiceResponse)
+ expect(response).not_to be_success
+ expect(response.http_status).to eq(expected_status)
+ end
+end
+
+RSpec.describe ::Projects::ImportExport::RelationImportService, :aggregate_failures, feature_category: :importers do
+ let_it_be(:project) { create(:project) }
+
+ let(:params) do
+ {
+ path: project_path,
+ file: fixture_file_upload('spec/features/projects/import_export/test_project_export.tar.gz', 'application/gzip'),
+ relation: relation
+ }
+ end
+
+ let(:relation) { 'issues' }
+
+ let_it_be(:user) { create(:user) }
+
+ subject(:import_service) { described_class.new(current_user: user, params: params) }
+
+ describe '#execute' do
+ context 'when the project exists' do
+ let(:project_path) { project.full_path }
+
+ context 'and the user is a maintainer' do
+ before_all do
+ project.add_maintainer(user)
+ end
+
+ it 'schedules a restore of the relation' do
+ expect(Projects::ImportExport::RelationImportWorker).to receive(:perform_async)
+
+ import_service.execute
+ end
+
+ it 'returns a service response' do
+ response = import_service.execute
+
+ expect(response).to be_instance_of(ServiceResponse)
+ expect(response).to be_success
+ expect(response.http_status).to eq(:ok)
+ expect(response.payload).to be_instance_of(Projects::ImportExport::RelationImportTracker)
+ end
+ end
+
+ context 'and the user has developer access' do
+ before_all do
+ project.add_developer(user)
+ end
+
+ include_examples 'returns failure response', :forbidden
+ end
+
+ context 'and the has no access' do
+ include_examples 'returns failure response', :forbidden
+ end
+
+ context 'and the user triggers an import before the last one finishes' do
+ before_all do
+ project.add_maintainer(user)
+ end
+
+ before do
+ project.relation_import_trackers.create!(relation: 'issues', status: 1)
+ end
+
+ include_examples 'returns failure response', :conflict
+ end
+
+ context 'and an invalid relation is passed' do
+ let(:relation) { 'invalid_relation' }
+
+ include_examples 'returns failure response', :bad_request
+ end
+ end
+
+ context 'when the project does not exist' do
+ let(:project_path) { 'some/unknown/project' }
+
+ include_examples 'returns failure response', :not_found
+ end
+ end
+end
diff --git a/spec/workers/every_sidekiq_worker_spec.rb b/spec/workers/every_sidekiq_worker_spec.rb
index 5597b8f16353a6b8f5326a5791b597e586f2705c..f42d56a819ba065a0d1ff54102842d85c605defd 100644
--- a/spec/workers/every_sidekiq_worker_spec.rb
+++ b/spec/workers/every_sidekiq_worker_spec.rb
@@ -410,6 +410,7 @@
'Projects::DeregisterSuggestedReviewersProjectWorker' => 3,
'Projects::DisableLegacyOpenSourceLicenseForInactiveProjectsWorker' => 3,
'Projects::GitGarbageCollectWorker' => false,
+ 'Projects::ImportExport::RelationImportWorker' => 6,
'Projects::InactiveProjectsDeletionNotificationWorker' => 3,
'Projects::PostCreationWorker' => 3,
'Projects::ScheduleBulkRepositoryShardMovesWorker' => 3,
diff --git a/spec/workers/projects/import_export/relation_import_worker_spec.rb b/spec/workers/projects/import_export/relation_import_worker_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..28bff08f57ac9818fa53112cd8232f7dabb6b087
--- /dev/null
+++ b/spec/workers/projects/import_export/relation_import_worker_spec.rb
@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Projects::ImportExport::RelationImportWorker, feature_category: :importers do
+ let_it_be_with_reload(:tracker) { create(:relation_import_tracker, status: 'created') }
+
+ let_it_be(:user) { create(:user) }
+ let(:worker) { described_class.new }
+
+ subject(:perform) { worker.perform(tracker.id, user.id) }
+
+ before do
+ tracker.project.update!(import_export_upload: create(
+ :import_export_upload,
+ import_file: fixture_file_upload('spec/features/projects/import_export/test_project_export.tar.gz')
+ ))
+ end
+
+ context 'when the import succeeds' do
+ it 'marks the tracker as finished' do
+ expect { perform }.to change { tracker.reload.finished? }.from(false).to(true)
+ end
+ end
+
+ context 'when the import fails' do
+ before do
+ allow(worker).to receive(:process_import).and_raise(StandardError, 'import_forced_to_fail')
+ end
+
+ it 'marks the tracker as failed' do
+ expect { perform }
+ .to raise_error(StandardError, 'import_forced_to_fail')
+ .and change { tracker.reload.failed? }.from(false).to(true)
+ end
+
+ it 'creates a record of the failure' do
+ expect { perform }
+ .to raise_error(StandardError, 'import_forced_to_fail')
+ .and change { tracker.reload.project.import_failures.count }.by(1)
+
+ failure = tracker.project.import_failures.last
+ expect(failure.exception_message).to eq('import_forced_to_fail')
+ end
+ end
+
+ it_behaves_like 'an idempotent worker' do
+ let(:job_args) { [tracker.id, user.id] }
+
+ it 'only starts one import when triggered multiple times' do
+ perform_multiple(job_args)
+
+ expect(tracker.reload.finished?).to be_truthy
+ end
+
+ it 'does not log any error when triggered multiple times' do
+ expect { perform_multiple(job_args) }.not_to change { tracker.reload.project.import_failures.count }
+ end
+ end
+
+ it_behaves_like 'worker with data consistency', described_class, data_consistency: :delayed
+end