Introduce Tasks migrated from old codebase

gems/gitlab-backup-cli/lib/gitlab/backup/cli.rb

@@ -13,6 +13,7 @@ module Cli
       autoload :Output, 'gitlab/backup/cli/output'
       autoload :Runner, 'gitlab/backup/cli/runner'
       autoload :Shell, 'gitlab/backup/cli/shell'
+      autoload :Tasks, 'gitlab/backup/cli/tasks'
       autoload :Utils, 'gitlab/backup/cli/utils'
       autoload :VERSION, 'gitlab/backup/cli/version'
 

gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Backup
+    module Cli
+      module Tasks
+        autoload :Artifacts, 'gitlab/backup/cli/tasks/artifacts'
+        autoload :Builds, 'gitlab/backup/cli/tasks/builds'
+        autoload :CiSecureFiles, 'gitlab/backup/cli/tasks/ci_secure_files'
+        autoload :Database, 'gitlab/backup/cli/tasks/database'
+        autoload :Lfs, 'gitlab/backup/cli/tasks/lfs'
+        autoload :Registry, 'gitlab/backup/cli/tasks/registry'
+        autoload :Repositories, 'gitlab/backup/cli/tasks/repositories'
+        autoload :Packages, 'gitlab/backup/cli/tasks/packages'
+        autoload :Pages, 'gitlab/backup/cli/tasks/pages'
+        autoload :Task, 'gitlab/backup/cli/tasks/task'
+        autoload :TerraformState, 'gitlab/backup/cli/tasks/terraform_state'
+        autoload :Uploads, 'gitlab/backup/cli/tasks/uploads'
+
+        TASKS = {
+          Gitlab::Backup::Cli::Tasks::Database.id => Gitlab::Backup::Cli::Tasks::Database,
+          Gitlab::Backup::Cli::Tasks::Repositories.id => Gitlab::Backup::Cli::Tasks::Repositories,
+          Gitlab::Backup::Cli::Tasks::Uploads.id => Gitlab::Backup::Cli::Tasks::Uploads,
+          Gitlab::Backup::Cli::Tasks::Builds.id => Gitlab::Backup::Cli::Tasks::Builds,
+          Gitlab::Backup::Cli::Tasks::Artifacts.id => Gitlab::Backup::Cli::Tasks::Artifacts,
+          Gitlab::Backup::Cli::Tasks::Pages.id => Gitlab::Backup::Cli::Tasks::Pages,
+          Gitlab::Backup::Cli::Tasks::Lfs.id => Gitlab::Backup::Cli::Tasks::Lfs,
+          Gitlab::Backup::Cli::Tasks::TerraformState.id => Gitlab::Backup::Cli::Tasks::TerraformState,
+          Gitlab::Backup::Cli::Tasks::Registry.id => Gitlab::Backup::Cli::Tasks::Registry,
+          Gitlab::Backup::Cli::Tasks::Packages.id => Gitlab::Backup::Cli::Tasks::Packages,
+          Gitlab::Backup::Cli::Tasks::CiSecureFiles.id => Gitlab::Backup::Cli::Tasks::CiSecureFiles
+        }.freeze
+
+        def self.all
+          TASKS.values
+        end
+      end
+    end
+  end
+end

gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/artifacts.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Backup
+    module Cli
+      module Tasks
+        class Artifacts < Task
+          def self.id = 'artifacts'
+
+          def human_name = _('artifacts')
+
+          def destination_path = 'artifacts.tar.gz'
+
+          private
+
+          def target
+            ::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: ['tmp'])
+          end
+
+          def storage_path
+            # TODO: Use configuration solver
+            JobArtifactUploader.root
+          end
+        end
+      end
+    end
+  end
+end

gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/builds.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Backup
+    module Cli
+      module Tasks
+        class Builds < Task
+          def self.id = 'builds'
+
+          def human_name = _('builds')
+
+          def destination_path = 'builds.tar.gz'
+
+          private
+
+          def target
+            ::Backup::Targets::Files.new(nil, storage_path, options: options)
+          end
+
+          def storage_path
+            # TODO: Use configuration solver
+            Settings.gitlab_ci.builds_path
+          end
+        end
+      end
+    end
+  end
+end

gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/ci_secure_files.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Backup
+    module Cli
+      module Tasks
+        class CiSecureFiles < Task
+          def self.id = 'ci_secure_files'
+
+          def human_name = _('ci secure files')
+
+          def destination_path = 'ci_secure_files.tar.gz'
+
+          private
+
+          def target
+            ::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: ['tmp'])
+          end
+
+          def storage_path
+            # TODO: Use configuration solver
+            Settings.ci_secure_files.storage_path
+          end
+        end
+      end
+    end
+  end
+end

gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/database.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Backup
+    module Cli
+      module Tasks
+        class Database < Task
+          def self.id = 'db'
+
+          def human_name = _('database')
+
+          def destination_path = 'db'
+
+          def cleanup_path = 'db'
+
+          private
+
+          def target
+            ::Backup::Targets::Database.new(output, options: options)
+          end
+        end
+      end
+    end
+  end
+end

gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/lfs.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Backup
+    module Cli
+      module Tasks
+        class Lfs < Task
+          def self.id = 'lfs'
+
+          def human_name = _('lfs objects')
+
+          def destination_path = 'lfs.tar.gz'
+
+          private
+
+          def target
+            ::Backup::Targets::Files.new(nil, storage_path, options: options)
+          end
+
+          def storage_path
+            # TODO: Use configuration solver
+            Settings.lfs.storage_path
+          end
+        end
+      end
+    end
+  end
+end

gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/packages.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Backup
+    module Cli
+      module Tasks
+        class Packages < Task
+          def self.id = 'packages'
+
+          def human_name = _('packages')
+
+          def destination_path = 'packages.tar.gz'
+
+          private
+
+          def target
+            ::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: ['tmp'])
+          end
+
+          def storage_path
+            # TODO: Use configuration solver
+            Settings.packages.storage_path
+          end
+        end
+      end
+    end
+  end
+end

gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/pages.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Backup
+    module Cli
+      module Tasks
+        class Pages < Task
+          # pages used to deploy tmp files to this path
+          # if some of these files are still there, we don't need them in the backup
+          LEGACY_PAGES_TMP_PATH = '@pages.tmp'
+
+          def self.id = 'pages'
+
+          def human_name = _('pages')
+
+          def destination_path = 'pages.tar.gz'
+
+          private
+
+          def target
+            ::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: [LEGACY_PAGES_TMP_PATH])
+          end
+
+          def storage_path
+            # TODO: Use configuration solver
+            Gitlab.config.pages.path
+          end
+        end
+      end
+    end
+  end
+end

gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/registry.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Backup
+    module Cli
+      module Tasks
+        class Registry < Task
+          def self.id = 'registry'
+
+          def enabled = Gitlab.config.registry.enabled
+
+          def human_name = _('container registry images')
+
+          def destination_path = 'registry.tar.gz'
+
+          private
+
+          def target
+            ::Backup::Targets::Files.new(nil, storage_path, options: options)
+          end
+
+          def storage_path
+            # TODO: Use configuration solver
+            Settings.registry.path
+          end
+        end
+      end
+    end
+  end
+end

gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/repositories.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Backup
+    module Cli
+      module Tasks
+        class Repositories < Task
+          def self.id = 'repositories'
+
+          def human_name = _('repositories')
+
+          def destination_path = 'repositories'
+
+          def destination_optional = true
+
+          private
+
+          def target
+            # TODO: migrate to the new codebase and rewrite portions to format output in a readable way
+            ::Backup::Targets::Repositories.new($stdout,
+              strategy: gitaly_strategy,
+              options: options,
+              storages: options.repositories_storages,
+              paths: options.repositories_paths,
+              skip_paths: options.skip_repositories_paths
+            )
+          end
+
+          def gitaly_strategy
+            # TODO: migrate to the new codebase and rewrite portions to format output in a readable way
+            ::Backup::GitalyBackup.new($stdout,
+              incremental: options.incremental?,
+              max_parallelism: options.max_parallelism,
+              storage_parallelism: options.max_storage_parallelism,
+              server_side: false
+            )
+          end
+        end
+      end
+    end
+  end
+end

gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/task.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Backup
+    module Cli
+      module Tasks
+        class Task
+          attr_reader :options, :context
+
+          # Identifier used as parameter in the CLI to skip from executing
+          def self.id
+            raise NotImplementedError
+          end
+
+          def initialize(context:, options:)
+            @context = context
+            @options = options
+          end
+
+          # Initiate a backup
+          #
+          # @param [Pathname] backup_path a path where to store the backups
+          # @param [String] backup_id
+          def backup!(backup_path, backup_id)
+            backup_output = backup_path.join(destination_path)
+
+            target.dump(backup_output, backup_id)
+          end
+
+          def restore!(backup_path, backup_id)
+            # TODO: enable this when building the restore functionality
+            # https://gitlab.com/gitlab-org/gitlab/-/issues/428505
+            raise NotImplementedError
+          end
+
+          # Key string that identifies the task
+          def id = self.class.id
+
+          # Name of the task used for logging.
+          def human_name
+            raise NotImplementedError
+          end
+
+          # Where the task should put its backup file/dir
+          def destination_path
+            raise NotImplementedError
+          end
+
+          # Path to remove after a successful backup, uses #destination_path when not specified
+          def cleanup_path
+            destination_path
+          end
+
+          # `true` if the destination might not exist on a successful backup
+          def destination_optional
+            false
+          end
+
+          # `true` if the task can be used
+          def enabled
+            true
+          end
+
+          def enabled?
+            enabled
+          end
+
+          private
+
+          # The target factory method
+          def target
+            raise NotImplementedError
+          end
+        end
+      end
+    end
+  end
+end

gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/terraform_state.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Backup
+    module Cli
+      module Tasks
+        class TerraformState < Task
+          def self.id = 'terraform_state'
+
+          def human_name = _('terraform states')
+
+          def destination_path = 'terraform_state.tar.gz'
+
+          private
+
+          def target
+            ::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: ['tmp'])
+          end
+
+          def storage_path
+            Settings.terraform_state.storage_path
+          end
+        end
+      end
+    end
+  end
+end

gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/uploads.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Backup
+    module Cli
+      module Tasks
+        class Uploads < Task
+          def self.id = 'uploads'
+
+          def human_name = _('uploads')
+
+          def destination_path = 'uploads.tar.gz'
+
+          private
+
+          def target
+            ::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: ['tmp'])
+          end
+
+          def storage_path
+            File.join(Gitlab.config.uploads.storage_path, 'uploads')
+          end
+        end
+      end
+    end
+  end
+end

gems/gitlab-backup-cli/spec/gitlab/backup/cli/tasks/task_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Backup::Cli::Tasks::Task do
+  let(:options) { nil }
+  let(:context) { build_fake_context }
+  let(:tmpdir) { Pathname.new(Dir.mktmpdir('task', temp_path)) }
+  let(:metadata) { build(:backup_metadata) }
+
+  subject(:task) { described_class.new(options: options, context: context) }
+
+  after do
+    FileUtils.rmtree(tmpdir)
+  end
+
+  context 'with unimplemented methods' do
+    describe '.id' do
+      it 'raises an error' do
+        expect { described_class.id }.to raise_error(NotImplementedError)
+      end
+    end
+
+    describe '#id' do
+      it 'raises an error' do
+        expect { task.id }.to raise_error(NotImplementedError)
+      end
+    end
+
+    describe '#human_name' do
+      it 'raises an error' do
+        expect { task.human_name }.to raise_error(NotImplementedError)
+      end
+    end
+
+    describe '#destination_path' do
+      it 'raises an error' do
+        expect { task.destination_path }.to raise_error(NotImplementedError)
+      end
+    end
+
+    describe '#target' do
+      it 'raises an error' do
+        expect { task.send(:target) }.to raise_error(NotImplementedError)
+      end
+    end
+  end
+
+  describe '#backup!' do
+    it 'delegates to target' do
+      expect(task).to receive(:destination_path).and_return(tmpdir.join('test_task'))
+      expect(task).to receive_message_chain(:target, :dump)
+
+      task.backup!(tmpdir, metadata.backup_id)
+    end
+  end
+end