diff --git a/app/controllers/projects_controller.rb b/app/controllers/projects_controller.rb
index 6b36bd40310f7d1db579f32a5a9acf1929bc0779..ed07971d4e126af7c617adb1a80aa458aed9af7e 100644
--- a/app/controllers/projects_controller.rb
+++ b/app/controllers/projects_controller.rb
@@ -3,6 +3,7 @@ class ProjectsController < Projects::ApplicationController
include ExtractsPath
include PreviewMarkdown
prepend EE::ProjectsController
+ include SendFileUpload
before_action :whitelist_query_limiting, only: [:create]
before_action :authenticate_user!, except: [:index, :show, :activity, :refs]
@@ -190,9 +191,9 @@ def export
end
def download_export
- export_project_path = @project.export_project_path
-
- if export_project_path
+ if export_project_object_storage?
+ send_upload(@project.import_export_upload.export_file)
+ elsif export_project_path
send_file export_project_path, disposition: 'attachment'
else
redirect_to(
@@ -267,8 +268,6 @@ def refs
render json: options.to_json
end
- private
-
# Render project landing depending of which features are available
# So if page is not availble in the list it renders the next page
#
@@ -433,4 +432,12 @@ def redirect_git_extension
def whitelist_query_limiting
Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42440')
end
+
+ def export_project_path
+ @export_project_path ||= @project.export_project_path
+ end
+
+ def export_project_object_storage?
+ @project.export_project_object_exists?
+ end
end
diff --git a/app/models/import_export_upload.rb b/app/models/import_export_upload.rb
new file mode 100644
index 0000000000000000000000000000000000000000..60d53d6c2c8a4beadb507fd959e4401c063bf42f
--- /dev/null
+++ b/app/models/import_export_upload.rb
@@ -0,0 +1,13 @@
+class ImportExportUpload < ActiveRecord::Base
+ include WithUploads
+ include ObjectStorage::BackgroundMove
+
+ belongs_to :project
+
+ mount_uploader :import_file, ImportExportUploader
+ mount_uploader :export_file, ImportExportUploader
+
+ def retrieve_upload(_identifier, paths)
+ Upload.find_by(model: self, path: paths)
+ end
+end
diff --git a/app/models/project.rb b/app/models/project.rb
index f88bda4483bd6840ecd75f0643568c0838ae17c1..2048f193620e3d2de409ec0d70c230d863c00eb4 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -176,6 +176,7 @@ class Project < ActiveRecord::Base
has_one :fork_network, through: :fork_network_member
has_one :import_state, autosave: true, class_name: 'ProjectImportState', inverse_of: :project
+ has_one :import_export_upload, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
# Merge Requests for target project should be removed with it
has_many :merge_requests, foreign_key: 'target_project_id'
@@ -1730,7 +1731,7 @@ def export_status
:started
elsif after_export_in_progress?
:after_export_action
- elsif export_project_path
+ elsif export_project_path || export_project_object_exists?
:finished
else
:none
@@ -1745,16 +1746,21 @@ def after_export_in_progress?
import_export_shared.after_export_in_progress?
end
- def remove_exports
- return nil unless export_path.present?
-
- FileUtils.rm_rf(export_path)
+ def remove_exports(path = export_path)
+ if path.present?
+ FileUtils.rm_rf(path)
+ elsif export_project_object_exists?
+ import_export_upload.remove_export_file!
+ import_export_upload.save
+ end
end
def remove_exported_project_file
- return unless export_project_path.present?
+ remove_exports(export_project_path)
+ end
- FileUtils.rm_f(export_project_path)
+ def export_project_object_exists?
+ Gitlab::ImportExport.object_storage? && import_export_upload&.export_file&.file
end
def full_path_slug
diff --git a/app/services/import_export_clean_up_service.rb b/app/services/import_export_clean_up_service.rb
index 74088b970c922496328337a4bc64ac1096278b51..3702c3742ef09bd63eade03b9dc32e435d335d05 100644
--- a/app/services/import_export_clean_up_service.rb
+++ b/app/services/import_export_clean_up_service.rb
@@ -10,7 +10,9 @@ def initialize(mmin = LAST_MODIFIED_TIME_IN_MINUTES)
def execute
Gitlab::Metrics.measure(:import_export_clean_up) do
- next unless File.directory?(path)
+ clean_up_export_object_files
+
+ break unless File.directory?(path)
clean_up_export_files
end
@@ -21,4 +23,11 @@ def execute
def clean_up_export_files
Gitlab::Popen.popen(%W(find #{path} -not -path #{path} -mmin +#{mmin} -delete))
end
+
+ def clean_up_export_object_files
+ ImportExportUpload.where('updated_at < ?', mmin.minutes.ago).each do |upload|
+ upload.remove_export_file!
+ upload.save!
+ end
+ end
end
diff --git a/app/uploaders/import_export_uploader.rb b/app/uploaders/import_export_uploader.rb
new file mode 100644
index 0000000000000000000000000000000000000000..213ac5c801197e49a276ff9f4c97657409f4b8a5
--- /dev/null
+++ b/app/uploaders/import_export_uploader.rb
@@ -0,0 +1,15 @@
+class ImportExportUploader < AttachmentUploader
+ EXTENSION_WHITELIST = %w[tar.gz].freeze
+
+ def extension_whitelist
+ EXTENSION_WHITELIST
+ end
+
+ def move_to_store
+ true
+ end
+
+ def move_to_cache
+ false
+ end
+end
diff --git a/app/views/projects/_export.html.haml b/app/views/projects/_export.html.haml
index f4d4888bd15e2e0e634741b750b0bd2984034b02..aa980da7e95a72ee5411d384b5e9025230de9a8a 100644
--- a/app/views/projects/_export.html.haml
+++ b/app/views/projects/_export.html.haml
@@ -31,7 +31,7 @@
%li Any encrypted tokens
%p
Once the exported file is ready, you will receive a notification email with a download link, or you can download it from this page.
- - if project.export_project_path
+ - if project.export_status == :finished
= link_to 'Download export', download_export_project_path(project),
rel: 'nofollow', download: '', method: :get, class: "btn btn-default"
= link_to 'Generate new export', generate_new_export_project_path(project),
diff --git a/changelogs/unreleased/46246-gitlab-project-export-should-use-object-storage.yml b/changelogs/unreleased/46246-gitlab-project-export-should-use-object-storage.yml
new file mode 100644
index 0000000000000000000000000000000000000000..908c7a238fd2e702ea24fcafe41f4e0d8e4c04c4
--- /dev/null
+++ b/changelogs/unreleased/46246-gitlab-project-export-should-use-object-storage.yml
@@ -0,0 +1,5 @@
+---
+title: Add Object Storage to project export
+merge_request: 20105
+author:
+type: added
diff --git a/db/migrate/20180625113853_create_import_export_uploads.rb b/db/migrate/20180625113853_create_import_export_uploads.rb
new file mode 100644
index 0000000000000000000000000000000000000000..be42304b0ae011f3ecf006163d59445ee05e131b
--- /dev/null
+++ b/db/migrate/20180625113853_create_import_export_uploads.rb
@@ -0,0 +1,16 @@
+class CreateImportExportUploads < ActiveRecord::Migration
+ DOWNTIME = false
+
+ def change
+ create_table :import_export_uploads do |t|
+ t.datetime_with_timezone :updated_at, null: false
+
+ t.references :project, index: true, foreign_key: { on_delete: :cascade }, unique: true
+
+ t.text :import_file
+ t.text :export_file
+ end
+
+ add_index :import_export_uploads, :updated_at
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 609c9d8077ed5f9cfa4205ef88317dd87b14d0c8..4c6ab3c85cacf52dcdd1a7675ab3b93eb96b5762 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -1312,6 +1312,16 @@
add_index "identities", ["saml_provider_id"], name: "index_identities_on_saml_provider_id", where: "(saml_provider_id IS NOT NULL)", using: :btree
add_index "identities", ["user_id"], name: "index_identities_on_user_id", using: :btree
+ create_table "import_export_uploads", force: :cascade do |t|
+ t.datetime_with_timezone "updated_at", null: false
+ t.integer "project_id"
+ t.text "import_file"
+ t.text "export_file"
+ end
+
+ add_index "import_export_uploads", ["project_id"], name: "index_import_export_uploads_on_project_id", using: :btree
+ add_index "import_export_uploads", ["updated_at"], name: "index_import_export_uploads_on_updated_at", using: :btree
+
create_table "index_statuses", force: :cascade do |t|
t.integer "project_id", null: false
t.datetime "indexed_at"
diff --git a/doc/administration/raketasks/project_import_export.md b/doc/administration/raketasks/project_import_export.md
index ecc4ac6b29bb21b1c40f8ac12f45ad50c4f44606..7bd765a35e02d297190d4307d35ad3653b1b8c0e 100644
--- a/doc/administration/raketasks/project_import_export.md
+++ b/doc/administration/raketasks/project_import_export.md
@@ -30,5 +30,12 @@ sudo gitlab-rake gitlab:import_export:data
bundle exec rake gitlab:import_export:data RAILS_ENV=production
```
+In order to enable Object Storage on the Export, you can use the [feature flag][feature-flags]:
+
+```
+import_export_object_storage
+```
+
[ce-3050]: https://gitlab.com/gitlab-org/gitlab-ce/issues/3050
+[feature-flags]: https://docs.gitlab.com/ee/api/features.html
[tmp]: ../../development/shared_files.md
diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb
index 5ef4e9d530c201e0b72517b8aa7ec280fe95ff77..15c57a2fc02e2dc69fb57ebe8cf2717ef96f3558 100644
--- a/lib/api/project_export.rb
+++ b/lib/api/project_export.rb
@@ -23,9 +23,13 @@ class ProjectExport < Grape::API
get ':id/export/download' do
path = user_project.export_project_path
- render_api_error!('404 Not found or has expired', 404) unless path
-
- present_disk_file!(path, File.basename(path), 'application/gzip')
+ if path
+ present_disk_file!(path, File.basename(path), 'application/gzip')
+ elsif user_project.export_project_object_exists?
+ present_carrierwave_file!(user_project.import_export_upload.export_file)
+ else
+ render_api_error!('404 Not found or has expired', 404)
+ end
end
desc 'Start export' do
diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb
index 53fe2f8e4361529b40cf6bee65d7fc4309d14641..be3710c5b7f7ae6894c9417c176e84e4663878c9 100644
--- a/lib/gitlab/import_export.rb
+++ b/lib/gitlab/import_export.rb
@@ -40,6 +40,10 @@ def export_filename(project:)
"#{basename[0..FILENAME_LIMIT]}_export.tar.gz"
end
+ def object_storage?
+ Feature.enabled?(:import_export_object_storage)
+ end
+
def version
VERSION
end
diff --git a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
index aef371d81ebce961140401392eeb2183775b5037..83134bb0769ef772fbb78b8e6ecf6e7ea492848e 100644
--- a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
+++ b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
@@ -2,6 +2,7 @@ module Gitlab
module ImportExport
module AfterExportStrategies
class BaseAfterExportStrategy
+ extend Gitlab::ImportExport::CommandLineUtil
include ActiveModel::Validations
extend Forwardable
@@ -24,9 +25,10 @@ def initialize(attributes = {})
end
def execute(current_user, project)
- return unless project&.export_project_path
-
@project = project
+
+ return unless @project.export_status == :finished
+
@current_user = current_user
if invalid?
@@ -51,9 +53,12 @@ def to_json(options = {})
end
def self.lock_file_path(project)
- return unless project&.export_path
+ return unless project.export_path || object_storage?
- File.join(project.export_path, AFTER_EXPORT_LOCK_FILE_NAME)
+ lock_path = project.import_export_shared.archive_path
+
+ mkdir_p(lock_path)
+ File.join(lock_path, AFTER_EXPORT_LOCK_FILE_NAME)
end
protected
@@ -77,6 +82,10 @@ def delete_after_export_lock
def log_validation_errors
errors.full_messages.each { |msg| project.import_export_shared.add_error_message(msg) }
end
+
+ def object_storage?
+ project.export_project_object_exists?
+ end
end
end
end
diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
index 938664a95a1503d1646bb5db9492b2df666c4fa7..dce8f89c0ab0b4edec88a405c7cd7192d8df2b96 100644
--- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
+++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
@@ -38,14 +38,20 @@ def handle_response_error(response)
private
def send_file
- export_file = File.open(project.export_project_path)
-
- Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options(export_file)) # rubocop:disable GitlabSecurity/PublicSend
+ Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options) # rubocop:disable GitlabSecurity/PublicSend
ensure
- export_file.close if export_file
+ export_file.close if export_file && !object_storage?
+ end
+
+ def export_file
+ if object_storage?
+ project.import_export_upload.export_file.file.open
+ else
+ File.open(project.export_project_path)
+ end
end
- def send_file_options(export_file)
+ def send_file_options
{
body_stream: export_file,
headers: headers
@@ -53,7 +59,15 @@ def send_file_options(export_file)
end
def headers
- { 'Content-Length' => File.size(project.export_project_path).to_s }
+ { 'Content-Length' => export_size.to_s }
+ end
+
+ def export_size
+ if object_storage?
+ project.import_export_upload.export_file.file.size
+ else
+ File.size(project.export_project_path)
+ end
end
end
end
diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb
index 2daeba90a5133dc890dba79b248cd2f23b7baefc..3cd153a4fd28bed89909c35065de6c06c64e6cc5 100644
--- a/lib/gitlab/import_export/saver.rb
+++ b/lib/gitlab/import_export/saver.rb
@@ -15,15 +15,22 @@ def initialize(project:, shared:)
def save
if compress_and_save
remove_export_path
+
Rails.logger.info("Saved project export #{archive_file}")
- archive_file
+
+ save_on_object_storage if use_object_storage?
else
- @shared.error(Gitlab::ImportExport::Error.new("Unable to save #{archive_file} into #{@shared.export_path}"))
+ @shared.error(Gitlab::ImportExport::Error.new(error_message))
false
end
rescue => e
@shared.error(e)
false
+ ensure
+ if use_object_storage?
+ remove_archive
+ remove_export_path
+ end
end
private
@@ -36,9 +43,29 @@ def remove_export_path
FileUtils.rm_rf(@shared.export_path)
end
+ def remove_archive
+ FileUtils.rm_rf(@shared.archive_path)
+ end
+
def archive_file
@archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project))
end
+
+ def save_on_object_storage
+ upload = ImportExportUpload.find_or_initialize_by(project: @project)
+
+ File.open(archive_file) { |file| upload.export_file = file }
+
+ upload.save!
+ end
+
+ def use_object_storage?
+ Gitlab::ImportExport.object_storage?
+ end
+
+ def error_message
+ "Unable to save #{archive_file} into #{@shared.export_path}. Object storage enabled: #{use_object_storage?}"
+ end
end
end
end
diff --git a/spec/controllers/projects_controller_spec.rb b/spec/controllers/projects_controller_spec.rb
index 560086aa486f2dd68b1690cbff858a1cc17ed64c..729dc21222d3733b748e4954c22d89de3d02b02b 100644
--- a/spec/controllers/projects_controller_spec.rb
+++ b/spec/controllers/projects_controller_spec.rb
@@ -835,23 +835,55 @@ def update_project(**parameters)
project.add_master(user)
end
- context 'when project export is enabled' do
- it 'returns 302' do
- get :download_export, namespace_id: project.namespace, id: project
+ context 'object storage disabled' do
+ before do
+ stub_feature_flags(import_export_object_storage: false)
+ end
- expect(response).to have_gitlab_http_status(302)
+ context 'when project export is enabled' do
+ it 'returns 302' do
+ get :download_export, namespace_id: project.namespace, id: project
+
+ expect(response).to have_gitlab_http_status(302)
+ end
+ end
+
+ context 'when project export is disabled' do
+ before do
+ stub_application_setting(project_export_enabled?: false)
+ end
+
+ it 'returns 404' do
+ get :download_export, namespace_id: project.namespace, id: project
+
+ expect(response).to have_gitlab_http_status(404)
+ end
end
end
- context 'when project export is disabled' do
+ context 'object storage enabled' do
before do
- stub_application_setting(project_export_enabled?: false)
+ stub_feature_flags(import_export_object_storage: true)
end
- it 'returns 404' do
- get :download_export, namespace_id: project.namespace, id: project
+ context 'when project export is enabled' do
+ it 'returns 302' do
+ get :download_export, namespace_id: project.namespace, id: project
- expect(response).to have_gitlab_http_status(404)
+ expect(response).to have_gitlab_http_status(302)
+ end
+ end
+
+ context 'when project export is disabled' do
+ before do
+ stub_application_setting(project_export_enabled?: false)
+ end
+
+ it 'returns 404' do
+ get :download_export, namespace_id: project.namespace, id: project
+
+ expect(response).to have_gitlab_http_status(404)
+ end
end
end
end
diff --git a/spec/factories/import_export_uploads.rb b/spec/factories/import_export_uploads.rb
new file mode 100644
index 0000000000000000000000000000000000000000..7750d49b1d05189367b0a4432a5a0e6a97e0b373
--- /dev/null
+++ b/spec/factories/import_export_uploads.rb
@@ -0,0 +1,5 @@
+FactoryBot.define do
+ factory :import_export_upload do
+ project { create(:project) }
+ end
+end
diff --git a/spec/factories/projects.rb b/spec/factories/projects.rb
index 5c8de6ba25a8254aaec115fc1621cbb4202e3936..a70712cced0c103bb4748d4bda4385fa9da6e3dd 100644
--- a/spec/factories/projects.rb
+++ b/spec/factories/projects.rb
@@ -141,6 +141,22 @@
end
trait :with_export do
+ before(:create) do |_project, _evaluator|
+ allow(Feature).to receive(:enabled?).with(:import_export_object_storage) { false }
+ allow(Feature).to receive(:enabled?).with('import_export_object_storage') { false }
+ end
+
+ after(:create) do |project, _evaluator|
+ ProjectExportWorker.new.perform(project.creator.id, project.id)
+ end
+ end
+
+ trait :with_object_export do
+ before(:create) do |_project, _evaluator|
+ allow(Feature).to receive(:enabled?).with(:import_export_object_storage) { true }
+ allow(Feature).to receive(:enabled?).with('import_export_object_storage') { true }
+ end
+
after(:create) do |project, evaluator|
ProjectExportWorker.new.perform(project.creator.id, project.id)
end
diff --git a/spec/features/projects/import_export/export_file_spec.rb b/spec/features/projects/import_export/export_file_spec.rb
index 8a4183565414241a74159669920174048389e412..eb281cd2122acf3bd047b17fc5dd83d43ca453e5 100644
--- a/spec/features/projects/import_export/export_file_spec.rb
+++ b/spec/features/projects/import_export/export_file_spec.rb
@@ -25,6 +25,7 @@
before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
+ stub_feature_flags(import_export_object_storage: false)
end
after do
diff --git a/spec/features/projects/import_export/namespace_export_file_spec.rb b/spec/features/projects/import_export/namespace_export_file_spec.rb
index 7d056b0c14006625d3db43e2931dd2bda8583e9d..9bb8a2063b516e41154a22279bc746800492c591 100644
--- a/spec/features/projects/import_export/namespace_export_file_spec.rb
+++ b/spec/features/projects/import_export/namespace_export_file_spec.rb
@@ -5,6 +5,7 @@
before do
allow(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
+ stub_feature_flags(import_export_object_storage: false)
end
after do
diff --git a/spec/fixtures/project_export.tar.gz b/spec/fixtures/project_export.tar.gz
new file mode 100644
index 0000000000000000000000000000000000000000..72ab2d71f355c1079020afa149105daf2533c199
Binary files /dev/null and b/spec/fixtures/project_export.tar.gz differ
diff --git a/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_object_storage_spec.rb b/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_object_storage_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..5059d68e54b52e8babc336bb16cece8b4a7d8f37
--- /dev/null
+++ b/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_object_storage_spec.rb
@@ -0,0 +1,105 @@
+require 'spec_helper'
+
+describe Gitlab::ImportExport::AfterExportStrategies::BaseAfterExportStrategy do
+ let!(:service) { described_class.new }
+ let!(:project) { create(:project, :with_object_export) }
+ let(:shared) { project.import_export_shared }
+ let!(:user) { create(:user) }
+
+ describe '#execute' do
+ before do
+ allow(service).to receive(:strategy_execute)
+ stub_feature_flags(import_export_object_storage: true)
+ end
+
+ it 'returns if project exported file is not found' do
+ allow(project).to receive(:export_project_object_exists?).and_return(false)
+
+ expect(service).not_to receive(:strategy_execute)
+
+ service.execute(user, project)
+ end
+
+ it 'creates a lock file in the export dir' do
+ allow(service).to receive(:delete_after_export_lock)
+
+ service.execute(user, project)
+
+ expect(lock_path_exist?).to be_truthy
+ end
+
+ context 'when the method succeeds' do
+ it 'removes the lock file' do
+ service.execute(user, project)
+
+ expect(lock_path_exist?).to be_falsey
+ end
+ end
+
+ context 'when the method fails' do
+ before do
+ allow(service).to receive(:strategy_execute).and_call_original
+ end
+
+ context 'when validation fails' do
+ before do
+ allow(service).to receive(:invalid?).and_return(true)
+ end
+
+ it 'does not create the lock file' do
+ expect(service).not_to receive(:create_or_update_after_export_lock)
+
+ service.execute(user, project)
+ end
+
+ it 'does not execute main logic' do
+ expect(service).not_to receive(:strategy_execute)
+
+ service.execute(user, project)
+ end
+
+ it 'logs validation errors in shared context' do
+ expect(service).to receive(:log_validation_errors)
+
+ service.execute(user, project)
+ end
+ end
+
+ context 'when an exception is raised' do
+ it 'removes the lock' do
+ expect { service.execute(user, project) }.to raise_error(NotImplementedError)
+
+ expect(lock_path_exist?).to be_falsey
+ end
+ end
+ end
+ end
+
+ describe '#log_validation_errors' do
+ it 'add the message to the shared context' do
+ errors = %w(test_message test_message2)
+
+ allow(service).to receive(:invalid?).and_return(true)
+ allow(service.errors).to receive(:full_messages).and_return(errors)
+
+ expect(shared).to receive(:add_error_message).twice.and_call_original
+
+ service.execute(user, project)
+
+ expect(shared.errors).to eq errors
+ end
+ end
+
+ describe '#to_json' do
+ it 'adds the current strategy class to the serialized attributes' do
+ params = { param1: 1 }
+ result = params.merge(klass: described_class.to_s).to_json
+
+ expect(described_class.new(params).to_json).to eq result
+ end
+ end
+
+ def lock_path_exist?
+ File.exist?(described_class.lock_file_path(project))
+ end
+end
diff --git a/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_spec.rb b/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_spec.rb
index ed54d87de4aa9121c0813c45902a603a3a8ca5b2..566b7f46c87e0c62233107354e5f1a4869214553 100644
--- a/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_spec.rb
+++ b/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_spec.rb
@@ -9,6 +9,7 @@
describe '#execute' do
before do
allow(service).to receive(:strategy_execute)
+ stub_feature_flags(import_export_object_storage: false)
end
it 'returns if project exported file is not found' do
diff --git a/spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb b/spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb
index 5fe57d9987b992c35e99a1e736712d42949c2f3c..7f2e0a4ee2c898e09d5c5e7b9fcea58abc3dc877 100644
--- a/spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb
+++ b/spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb
@@ -24,13 +24,34 @@
end
describe '#execute' do
- it 'removes the exported project file after the upload' do
- allow(strategy).to receive(:send_file)
- allow(strategy).to receive(:handle_response_error)
+ context 'without object storage' do
+ before do
+ stub_feature_flags(import_export_object_storage: false)
+ end
+
+ it 'removes the exported project file after the upload' do
+ allow(strategy).to receive(:send_file)
+ allow(strategy).to receive(:handle_response_error)
+
+ expect(project).to receive(:remove_exported_project_file)
+
+ strategy.execute(user, project)
+ end
+ end
+
+ context 'with object storage' do
+ before do
+ stub_feature_flags(import_export_object_storage: true)
+ end
- expect(project).to receive(:remove_exported_project_file)
+ it 'removes the exported project file after the upload' do
+ allow(strategy).to receive(:send_file)
+ allow(strategy).to receive(:handle_response_error)
- strategy.execute(user, project)
+ expect(project).to receive(:remove_exported_project_file)
+
+ strategy.execute(user, project)
+ end
end
end
end
diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml
index 475a3011c8f2ddba948a714c9ec29782182d358a..2ba1377ebdb884361a177a43832d12de3c6212b2 100644
--- a/spec/lib/gitlab/import_export/all_models.yml
+++ b/spec/lib/gitlab/import_export/all_models.yml
@@ -330,6 +330,7 @@ project:
- deploy_tokens
- settings
- ci_cd_settings
+- import_export_upload
- vulnerability_feedback
award_emoji:
- awardable
diff --git a/spec/lib/gitlab/import_export/saver_spec.rb b/spec/lib/gitlab/import_export/saver_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..02f1a4b81aa80af52f7d859a1bd1a6a812a308a4
--- /dev/null
+++ b/spec/lib/gitlab/import_export/saver_spec.rb
@@ -0,0 +1,43 @@
+require 'spec_helper'
+require 'fileutils'
+
+describe Gitlab::ImportExport::Saver do
+ let!(:project) { create(:project, :public, name: 'project') }
+ let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" }
+ let(:shared) { project.import_export_shared }
+ subject { described_class.new(project: project, shared: shared) }
+
+ before do
+ allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
+
+ FileUtils.mkdir_p(shared.export_path)
+ FileUtils.touch("#{shared.export_path}/tmp.bundle")
+ end
+
+ after do
+ FileUtils.rm_rf(export_path)
+ end
+
+ context 'local archive' do
+ it 'saves the repo to disk' do
+ stub_feature_flags(import_export_object_storage: false)
+
+ subject.save
+
+ expect(shared.errors).to be_empty
+ expect(Dir.empty?(shared.archive_path)).to be false
+ end
+ end
+
+ context 'object storage' do
+ it 'saves the repo using object storage' do
+ stub_feature_flags(import_export_object_storage: true)
+ stub_uploads_object_storage(ImportExportUploader)
+
+ subject.save
+
+ expect(ImportExportUpload.find_by(project: project).export_file.url)
+ .to match(%r[\/uploads\/-\/system\/import_export_upload\/export_file.*])
+ end
+ end
+end
diff --git a/spec/models/import_export_upload_spec.rb b/spec/models/import_export_upload_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..58af84b8a089cb497af89079fea3c4a5de23c4a4
--- /dev/null
+++ b/spec/models/import_export_upload_spec.rb
@@ -0,0 +1,25 @@
+require 'spec_helper'
+
+describe ImportExportUpload do
+ subject { described_class.new(project: create(:project)) }
+
+ shared_examples 'stores the Import/Export file' do |method|
+ it 'stores the import file' do
+ subject.public_send("#{method}=", fixture_file_upload('spec/fixtures/project_export.tar.gz'))
+
+ subject.save!
+
+ url = "/uploads/-/system/import_export_upload/#{method}/#{subject.id}/project_export.tar.gz"
+
+ expect(subject.public_send(method).url).to eq(url)
+ end
+ end
+
+ context 'import' do
+ it_behaves_like 'stores the Import/Export file', :import_file
+ end
+
+ context 'export' do
+ it_behaves_like 'stores the Import/Export file', :export_file
+ end
+end
diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb
index c34f410cc6e909aaf35fa3fb98b57dc3e6eefb3f..75f1da673eb917903538488841d0c3362a242d4f 100644
--- a/spec/models/project_spec.rb
+++ b/spec/models/project_spec.rb
@@ -3131,6 +3131,10 @@ def enable_lfs
let(:legacy_project) { create(:project, :legacy_storage, :with_export) }
let(:project) { create(:project, :with_export) }
+ before do
+ stub_feature_flags(import_export_object_storage: false)
+ end
+
it 'removes the exports directory for the project' do
expect(File.exist?(project.export_path)).to be_truthy
@@ -3179,12 +3183,14 @@ def enable_lfs
let(:project) { create(:project, :with_export) }
it 'removes the exported project file' do
+ stub_feature_flags(import_export_object_storage: false)
+
exported_file = project.export_project_path
expect(File.exist?(exported_file)).to be_truthy
- allow(FileUtils).to receive(:rm_f).and_call_original
- expect(FileUtils).to receive(:rm_f).with(exported_file).and_call_original
+ allow(FileUtils).to receive(:rm_rf).and_call_original
+ expect(FileUtils).to receive(:rm_rf).with(exported_file).and_call_original
project.remove_exported_project_file
diff --git a/spec/requests/api/project_export_spec.rb b/spec/requests/api/project_export_spec.rb
index 3834d27d0a9b0da56a7e6a7477906ef91cb2d7cb..a4615bd081f5fa12a6c81b9155fb023f1b600db6 100644
--- a/spec/requests/api/project_export_spec.rb
+++ b/spec/requests/api/project_export_spec.rb
@@ -192,6 +192,13 @@
context 'when upload complete' do
before do
FileUtils.rm_rf(project_after_export.export_path)
+
+ if project_after_export.export_project_object_exists?
+ upload = project_after_export.import_export_upload
+
+ upload.remove_export_file!
+ upload.save
+ end
end
it_behaves_like '404 response' do
@@ -261,6 +268,22 @@
it_behaves_like 'get project export download not found'
end
end
+
+ context 'when an uploader is used' do
+ before do
+ stub_uploads_object_storage(ImportExportUploader)
+
+ [project, project_finished, project_after_export].each do |p|
+ p.add_master(user)
+
+ upload = ImportExportUpload.new(project: p)
+ upload.export_file = fixture_file_upload('spec/fixtures/project_export.tar.gz', "`/tar.gz")
+ upload.save!
+ end
+ end
+
+ it_behaves_like 'get project download by strategy'
+ end
end
describe 'POST /projects/:project_id/export' do
diff --git a/spec/services/import_export_clean_up_service_spec.rb b/spec/services/import_export_clean_up_service_spec.rb
index 1875d0448cd5943ea5dc358cd5da4b9874687a1c..d5fcef1246f2808f382379b7320630b53e7245e2 100644
--- a/spec/services/import_export_clean_up_service_spec.rb
+++ b/spec/services/import_export_clean_up_service_spec.rb
@@ -11,7 +11,6 @@
path = '/invalid/path/'
stub_repository_downloads_path(path)
- expect(File).to receive(:directory?).with(path + tmp_import_export_folder).and_return(false).at_least(:once)
expect(service).not_to receive(:clean_up_export_files)
service.execute
@@ -38,6 +37,24 @@
end
end
+ context 'with uploader exports' do
+ it 'removes old files' do
+ upload = create(:import_export_upload,
+ updated_at: 2.days.ago,
+ export_file: fixture_file_upload('spec/fixtures/project_export.tar.gz'))
+
+ expect { service.execute }.to change { upload.reload.export_file.file.nil? }.to(true)
+ end
+
+ it 'does not remove new files' do
+ upload = create(:import_export_upload,
+ updated_at: 1.hour.ago,
+ export_file: fixture_file_upload('spec/fixtures/project_export.tar.gz'))
+
+ expect { service.execute }.not_to change { upload.reload.export_file.file.nil? }
+ end
+ end
+
def in_directory_with_files(mtime:)
Dir.mktmpdir do |tmpdir|
stub_repository_downloads_path(tmpdir)
diff --git a/spec/uploaders/import_export_uploader_spec.rb b/spec/uploaders/import_export_uploader_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..51b173b682dafaf28e1a79513a1e26fa20f8b7c9
--- /dev/null
+++ b/spec/uploaders/import_export_uploader_spec.rb
@@ -0,0 +1,20 @@
+require 'spec_helper'
+
+describe ImportExportUploader do
+ let(:model) { build_stubbed(:import_export_upload) }
+ let(:upload) { create(:upload, model: model) }
+
+ subject { described_class.new(model, :import_file) }
+
+ context "object_store is REMOTE" do
+ before do
+ stub_uploads_object_storage
+ end
+
+ include_context 'with storage', described_class::Store::REMOTE
+
+ it_behaves_like 'builds correct paths',
+ store_dir: %r[import_export_upload/import_file/],
+ upload_path: %r[import_export_upload/import_file/]
+ end
+end