From 911db940d92192e975c85d077ac754bfd84529fa Mon Sep 17 00:00:00 2001
From: Gabriel Mazetto <gabriel@gitlab.com>
Date: Mon, 26 Feb 2024 00:38:05 +0100
Subject: [PATCH] Introduce Tasks migrated from old codebase
---
.../lib/gitlab/backup/cli.rb | 1 +
.../lib/gitlab/backup/cli/tasks.rb | 40 ++++++++++
.../lib/gitlab/backup/cli/tasks/artifacts.rb | 28 +++++++
.../lib/gitlab/backup/cli/tasks/builds.rb | 28 +++++++
.../backup/cli/tasks/ci_secure_files.rb | 28 +++++++
.../lib/gitlab/backup/cli/tasks/database.rb | 25 ++++++
.../lib/gitlab/backup/cli/tasks/lfs.rb | 28 +++++++
.../lib/gitlab/backup/cli/tasks/packages.rb | 28 +++++++
.../lib/gitlab/backup/cli/tasks/pages.rb | 32 ++++++++
.../lib/gitlab/backup/cli/tasks/registry.rb | 30 +++++++
.../gitlab/backup/cli/tasks/repositories.rb | 42 ++++++++++
.../lib/gitlab/backup/cli/tasks/task.rb | 78 +++++++++++++++++++
.../backup/cli/tasks/terraform_state.rb | 27 +++++++
.../lib/gitlab/backup/cli/tasks/uploads.rb | 27 +++++++
.../spec/gitlab/backup/cli/tasks/task_spec.rb | 57 ++++++++++++++
15 files changed, 499 insertions(+)
create mode 100644 gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks.rb
create mode 100644 gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/artifacts.rb
create mode 100644 gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/builds.rb
create mode 100644 gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/ci_secure_files.rb
create mode 100644 gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/database.rb
create mode 100644 gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/lfs.rb
create mode 100644 gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/packages.rb
create mode 100644 gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/pages.rb
create mode 100644 gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/registry.rb
create mode 100644 gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/repositories.rb
create mode 100644 gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/task.rb
create mode 100644 gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/terraform_state.rb
create mode 100644 gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/uploads.rb
create mode 100644 gems/gitlab-backup-cli/spec/gitlab/backup/cli/tasks/task_spec.rb
diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli.rb
index da939191df40..7259107a9176 100644
--- a/gems/gitlab-backup-cli/lib/gitlab/backup/cli.rb
+++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli.rb
@@ -13,6 +13,7 @@ module Cli
autoload :Output, 'gitlab/backup/cli/output'
autoload :Runner, 'gitlab/backup/cli/runner'
autoload :Shell, 'gitlab/backup/cli/shell'
+ autoload :Tasks, 'gitlab/backup/cli/tasks'
autoload :Utils, 'gitlab/backup/cli/utils'
autoload :VERSION, 'gitlab/backup/cli/version'
diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks.rb
new file mode 100644
index 000000000000..d5635c7a5265
--- /dev/null
+++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Backup
+ module Cli
+ module Tasks
+ autoload :Artifacts, 'gitlab/backup/cli/tasks/artifacts'
+ autoload :Builds, 'gitlab/backup/cli/tasks/builds'
+ autoload :CiSecureFiles, 'gitlab/backup/cli/tasks/ci_secure_files'
+ autoload :Database, 'gitlab/backup/cli/tasks/database'
+ autoload :Lfs, 'gitlab/backup/cli/tasks/lfs'
+ autoload :Registry, 'gitlab/backup/cli/tasks/registry'
+ autoload :Repositories, 'gitlab/backup/cli/tasks/repositories'
+ autoload :Packages, 'gitlab/backup/cli/tasks/packages'
+ autoload :Pages, 'gitlab/backup/cli/tasks/pages'
+ autoload :Task, 'gitlab/backup/cli/tasks/task'
+ autoload :TerraformState, 'gitlab/backup/cli/tasks/terraform_state'
+ autoload :Uploads, 'gitlab/backup/cli/tasks/uploads'
+
+ TASKS = {
+ Gitlab::Backup::Cli::Tasks::Database.id => Gitlab::Backup::Cli::Tasks::Database,
+ Gitlab::Backup::Cli::Tasks::Repositories.id => Gitlab::Backup::Cli::Tasks::Repositories,
+ Gitlab::Backup::Cli::Tasks::Uploads.id => Gitlab::Backup::Cli::Tasks::Uploads,
+ Gitlab::Backup::Cli::Tasks::Builds.id => Gitlab::Backup::Cli::Tasks::Builds,
+ Gitlab::Backup::Cli::Tasks::Artifacts.id => Gitlab::Backup::Cli::Tasks::Artifacts,
+ Gitlab::Backup::Cli::Tasks::Pages.id => Gitlab::Backup::Cli::Tasks::Pages,
+ Gitlab::Backup::Cli::Tasks::Lfs.id => Gitlab::Backup::Cli::Tasks::Lfs,
+ Gitlab::Backup::Cli::Tasks::TerraformState.id => Gitlab::Backup::Cli::Tasks::TerraformState,
+ Gitlab::Backup::Cli::Tasks::Registry.id => Gitlab::Backup::Cli::Tasks::Registry,
+ Gitlab::Backup::Cli::Tasks::Packages.id => Gitlab::Backup::Cli::Tasks::Packages,
+ Gitlab::Backup::Cli::Tasks::CiSecureFiles.id => Gitlab::Backup::Cli::Tasks::CiSecureFiles
+ }.freeze
+
+ def self.all
+ TASKS.values
+ end
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/artifacts.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/artifacts.rb
new file mode 100644
index 000000000000..80620c536d42
--- /dev/null
+++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/artifacts.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Backup
+ module Cli
+ module Tasks
+ class Artifacts < Task
+ def self.id = 'artifacts'
+
+ def human_name = _('artifacts')
+
+ def destination_path = 'artifacts.tar.gz'
+
+ private
+
+ def target
+ ::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: ['tmp'])
+ end
+
+ def storage_path
+ # TODO: Use configuration solver
+ JobArtifactUploader.root
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/builds.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/builds.rb
new file mode 100644
index 000000000000..f2f75008fa97
--- /dev/null
+++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/builds.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Backup
+ module Cli
+ module Tasks
+ class Builds < Task
+ def self.id = 'builds'
+
+ def human_name = _('builds')
+
+ def destination_path = 'builds.tar.gz'
+
+ private
+
+ def target
+ ::Backup::Targets::Files.new(nil, storage_path, options: options)
+ end
+
+ def storage_path
+ # TODO: Use configuration solver
+ Settings.gitlab_ci.builds_path
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/ci_secure_files.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/ci_secure_files.rb
new file mode 100644
index 000000000000..7c4b7d8a2ca6
--- /dev/null
+++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/ci_secure_files.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Backup
+ module Cli
+ module Tasks
+ class CiSecureFiles < Task
+ def self.id = 'ci_secure_files'
+
+ def human_name = _('ci secure files')
+
+ def destination_path = 'ci_secure_files.tar.gz'
+
+ private
+
+ def target
+ ::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: ['tmp'])
+ end
+
+ def storage_path
+ # TODO: Use configuration solver
+ Settings.ci_secure_files.storage_path
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/database.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/database.rb
new file mode 100644
index 000000000000..b67b2af958ac
--- /dev/null
+++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/database.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Backup
+ module Cli
+ module Tasks
+ class Database < Task
+ def self.id = 'db'
+
+ def human_name = _('database')
+
+ def destination_path = 'db'
+
+ def cleanup_path = 'db'
+
+ private
+
+ def target
+ ::Backup::Targets::Database.new(output, options: options)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/lfs.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/lfs.rb
new file mode 100644
index 000000000000..3ed3d47baec5
--- /dev/null
+++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/lfs.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Backup
+ module Cli
+ module Tasks
+ class Lfs < Task
+ def self.id = 'lfs'
+
+ def human_name = _('lfs objects')
+
+ def destination_path = 'lfs.tar.gz'
+
+ private
+
+ def target
+ ::Backup::Targets::Files.new(nil, storage_path, options: options)
+ end
+
+ def storage_path
+ # TODO: Use configuration solver
+ Settings.lfs.storage_path
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/packages.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/packages.rb
new file mode 100644
index 000000000000..398f93af0406
--- /dev/null
+++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/packages.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Backup
+ module Cli
+ module Tasks
+ class Packages < Task
+ def self.id = 'packages'
+
+ def human_name = _('packages')
+
+ def destination_path = 'packages.tar.gz'
+
+ private
+
+ def target
+ ::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: ['tmp'])
+ end
+
+ def storage_path
+ # TODO: Use configuration solver
+ Settings.packages.storage_path
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/pages.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/pages.rb
new file mode 100644
index 000000000000..4c44ab600b32
--- /dev/null
+++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/pages.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Backup
+ module Cli
+ module Tasks
+ class Pages < Task
+ # pages used to deploy tmp files to this path
+ # if some of these files are still there, we don't need them in the backup
+ LEGACY_PAGES_TMP_PATH = '@pages.tmp'
+
+ def self.id = 'pages'
+
+ def human_name = _('pages')
+
+ def destination_path = 'pages.tar.gz'
+
+ private
+
+ def target
+ ::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: [LEGACY_PAGES_TMP_PATH])
+ end
+
+ def storage_path
+ # TODO: Use configuration solver
+ Gitlab.config.pages.path
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/registry.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/registry.rb
new file mode 100644
index 000000000000..ee90782ce54d
--- /dev/null
+++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/registry.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Backup
+ module Cli
+ module Tasks
+ class Registry < Task
+ def self.id = 'registry'
+
+ def enabled = Gitlab.config.registry.enabled
+
+ def human_name = _('container registry images')
+
+ def destination_path = 'registry.tar.gz'
+
+ private
+
+ def target
+ ::Backup::Targets::Files.new(nil, storage_path, options: options)
+ end
+
+ def storage_path
+ # TODO: Use configuration solver
+ Settings.registry.path
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/repositories.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/repositories.rb
new file mode 100644
index 000000000000..f4d172d623e5
--- /dev/null
+++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/repositories.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Backup
+ module Cli
+ module Tasks
+ class Repositories < Task
+ def self.id = 'repositories'
+
+ def human_name = _('repositories')
+
+ def destination_path = 'repositories'
+
+ def destination_optional = true
+
+ private
+
+ def target
+ # TODO: migrate to the new codebase and rewrite portions to format output in a readable way
+ ::Backup::Targets::Repositories.new($stdout,
+ strategy: gitaly_strategy,
+ options: options,
+ storages: options.repositories_storages,
+ paths: options.repositories_paths,
+ skip_paths: options.skip_repositories_paths
+ )
+ end
+
+ def gitaly_strategy
+ # TODO: migrate to the new codebase and rewrite portions to format output in a readable way
+ ::Backup::GitalyBackup.new($stdout,
+ incremental: options.incremental?,
+ max_parallelism: options.max_parallelism,
+ storage_parallelism: options.max_storage_parallelism,
+ server_side: false
+ )
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/task.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/task.rb
new file mode 100644
index 000000000000..995703ee1f1a
--- /dev/null
+++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/task.rb
@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Backup
+ module Cli
+ module Tasks
+ class Task
+ attr_reader :options, :context
+
+ # Identifier used as parameter in the CLI to skip from executing
+ def self.id
+ raise NotImplementedError
+ end
+
+ def initialize(context:, options:)
+ @context = context
+ @options = options
+ end
+
+ # Initiate a backup
+ #
+ # @param [Pathname] backup_path a path where to store the backups
+ # @param [String] backup_id
+ def backup!(backup_path, backup_id)
+ backup_output = backup_path.join(destination_path)
+
+ target.dump(backup_output, backup_id)
+ end
+
+ def restore!(backup_path, backup_id)
+ # TODO: enable this when building the restore functionality
+ # https://gitlab.com/gitlab-org/gitlab/-/issues/428505
+ raise NotImplementedError
+ end
+
+ # Key string that identifies the task
+ def id = self.class.id
+
+ # Name of the task used for logging.
+ def human_name
+ raise NotImplementedError
+ end
+
+ # Where the task should put its backup file/dir
+ def destination_path
+ raise NotImplementedError
+ end
+
+ # Path to remove after a successful backup, uses #destination_path when not specified
+ def cleanup_path
+ destination_path
+ end
+
+ # `true` if the destination might not exist on a successful backup
+ def destination_optional
+ false
+ end
+
+ # `true` if the task can be used
+ def enabled
+ true
+ end
+
+ def enabled?
+ enabled
+ end
+
+ private
+
+ # The target factory method
+ def target
+ raise NotImplementedError
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/terraform_state.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/terraform_state.rb
new file mode 100644
index 000000000000..8504fd19bd8e
--- /dev/null
+++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/terraform_state.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Backup
+ module Cli
+ module Tasks
+ class TerraformState < Task
+ def self.id = 'terraform_state'
+
+ def human_name = _('terraform states')
+
+ def destination_path = 'terraform_state.tar.gz'
+
+ private
+
+ def target
+ ::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: ['tmp'])
+ end
+
+ def storage_path
+ Settings.terraform_state.storage_path
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/uploads.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/uploads.rb
new file mode 100644
index 000000000000..6ca765e3ad2d
--- /dev/null
+++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/uploads.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Backup
+ module Cli
+ module Tasks
+ class Uploads < Task
+ def self.id = 'uploads'
+
+ def human_name = _('uploads')
+
+ def destination_path = 'uploads.tar.gz'
+
+ private
+
+ def target
+ ::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: ['tmp'])
+ end
+
+ def storage_path
+ File.join(Gitlab.config.uploads.storage_path, 'uploads')
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-backup-cli/spec/gitlab/backup/cli/tasks/task_spec.rb b/gems/gitlab-backup-cli/spec/gitlab/backup/cli/tasks/task_spec.rb
new file mode 100644
index 000000000000..e578ff11740b
--- /dev/null
+++ b/gems/gitlab-backup-cli/spec/gitlab/backup/cli/tasks/task_spec.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Backup::Cli::Tasks::Task do
+ let(:options) { nil }
+ let(:context) { build_fake_context }
+ let(:tmpdir) { Pathname.new(Dir.mktmpdir('task', temp_path)) }
+ let(:metadata) { build(:backup_metadata) }
+
+ subject(:task) { described_class.new(options: options, context: context) }
+
+ after do
+ FileUtils.rmtree(tmpdir)
+ end
+
+ context 'with unimplemented methods' do
+ describe '.id' do
+ it 'raises an error' do
+ expect { described_class.id }.to raise_error(NotImplementedError)
+ end
+ end
+
+ describe '#id' do
+ it 'raises an error' do
+ expect { task.id }.to raise_error(NotImplementedError)
+ end
+ end
+
+ describe '#human_name' do
+ it 'raises an error' do
+ expect { task.human_name }.to raise_error(NotImplementedError)
+ end
+ end
+
+ describe '#destination_path' do
+ it 'raises an error' do
+ expect { task.destination_path }.to raise_error(NotImplementedError)
+ end
+ end
+
+ describe '#target' do
+ it 'raises an error' do
+ expect { task.send(:target) }.to raise_error(NotImplementedError)
+ end
+ end
+ end
+
+ describe '#backup!' do
+ it 'delegates to target' do
+ expect(task).to receive(:destination_path).and_return(tmpdir.join('test_task'))
+ expect(task).to receive_message_chain(:target, :dump)
+
+ task.backup!(tmpdir, metadata.backup_id)
+ end
+ end
+end
--
GitLab