From d76a302def4dd678b99389e286d15aebde3d9c62 Mon Sep 17 00:00:00 2001
From: Vladimir Shushlin <vshushlin@gitlab.com>
Date: Mon, 2 Oct 2023 15:46:13 +0000
Subject: [PATCH] Add rake task for resetting unreadable encrypted tokens
We have runners_tokens in DB that can not be decrypted.
Automatically regenerating them is dangerous,
so we provide admins with the rake task to do that.
---
lib/gitlab/doctor/reset_tokens.rb | 66 +++++++++
lib/tasks/gitlab/doctor/secrets.rake | 16 +++
spec/lib/gitlab/doctor/reset_tokens_spec.rb | 133 ++++++++++++++++++
spec/tasks/gitlab/doctor/secrets_rake_spec.rb | 56 ++++++++
4 files changed, 271 insertions(+)
create mode 100644 lib/gitlab/doctor/reset_tokens.rb
create mode 100644 spec/lib/gitlab/doctor/reset_tokens_spec.rb
create mode 100644 spec/tasks/gitlab/doctor/secrets_rake_spec.rb
diff --git a/lib/gitlab/doctor/reset_tokens.rb b/lib/gitlab/doctor/reset_tokens.rb
new file mode 100644
index 0000000000000..45333e2effb93
--- /dev/null
+++ b/lib/gitlab/doctor/reset_tokens.rb
@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Doctor
+ class ResetTokens
+ attr_reader :logger
+
+ PRINT_PROGRESS_EVERY = 1000
+
+ def initialize(logger, model_names:, token_names:, dry_run: true)
+ @logger = logger
+ @model_names = model_names
+ @token_names = token_names
+ @dry_run = dry_run
+ end
+
+ def run!
+ logger.info "Resetting #{@token_names.join(', ')} on #{@model_names.join(', ')} if they can not be read"
+ logger.info "Executing in DRY RUN mode, no records will actually be updated" if @dry_run
+ Rails.application.eager_load!
+
+ models_with_encrypted_tokens.each do |model|
+ fix_model(model)
+ end
+ logger.info "Done!"
+ end
+
+ private
+
+ def fix_model(model)
+ matched_token_names = @token_names & model.encrypted_token_authenticatable_fields.map(&:to_s)
+
+ return if matched_token_names.empty?
+
+ total_count = model.count
+
+ model.find_each.with_index do |instance, index|
+ matched_token_names.each do |attribute_name|
+ fix_attribute(instance, attribute_name)
+ end
+
+ logger.info "Checked #{index + 1}/#{total_count} #{model.name.pluralize}" if index % PRINT_PROGRESS_EVERY == 0
+ end
+ logger.info "Checked #{total_count} #{model.name.pluralize}"
+ end
+
+ def fix_attribute(instance, attribute_name)
+ instance.public_send(attribute_name) # rubocop:disable GitlabSecurity/PublicSend
+ rescue OpenSSL::Cipher::CipherError, TypeError
+ logger.debug "> Fix #{instance.class.name}[#{instance.id}].#{attribute_name}"
+ instance.public_send("reset_#{attribute_name}!") unless @dry_run # rubocop:disable GitlabSecurity/PublicSend
+ rescue StandardError => e
+ logger.debug(
+ "> Something went wrong for #{instance.class.name}[#{instance.id}].#{attribute_name}: #{e}".color(:red))
+
+ false
+ end
+
+ def models_with_encrypted_tokens
+ ApplicationRecord.descendants.select do |model|
+ @model_names.include?(model.name) && model.include?(TokenAuthenticatable)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/tasks/gitlab/doctor/secrets.rake b/lib/tasks/gitlab/doctor/secrets.rake
index 29f0f36c705d0..dd005005edf29 100644
--- a/lib/tasks/gitlab/doctor/secrets.rake
+++ b/lib/tasks/gitlab/doctor/secrets.rake
@@ -10,5 +10,21 @@ namespace :gitlab do
Gitlab::Doctor::Secrets.new(logger).run!
end
+
+ desc "GitLab | Reset encrypted tokens for specific models"
+ task reset_encrypted_tokens: :gitlab_environment do
+ logger = Logger.new($stdout)
+
+ logger.level = Gitlab::Utils.to_boolean(ENV['VERBOSE']) ? Logger::DEBUG : Logger::INFO
+ model_names = ENV['MODEL_NAMES']&.split(',')
+ token_names = ENV['TOKEN_NAMES']&.split(',')
+ dry_run = Gitlab::Utils.to_boolean(ENV['DRY_RUN'])
+ dry_run = true if dry_run.nil?
+
+ next logger.info("No models were specified, please use MODEL_NAMES environment variable") unless model_names
+ next logger.info("No tokens were specified, please use TOKEN_NAMES environment variable") unless token_names
+
+ Gitlab::Doctor::ResetTokens.new(logger, model_names: model_names, token_names: token_names, dry_run: dry_run).run!
+ end
end
end
diff --git a/spec/lib/gitlab/doctor/reset_tokens_spec.rb b/spec/lib/gitlab/doctor/reset_tokens_spec.rb
new file mode 100644
index 0000000000000..0cc947efdb479
--- /dev/null
+++ b/spec/lib/gitlab/doctor/reset_tokens_spec.rb
@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Doctor::ResetTokens, feature_category: :runner_fleet do
+ let(:logger) { instance_double('Logger') }
+ let(:model_names) { %w[Project Group] }
+ let(:token_names) { %w[runners_token] }
+ let(:dry_run) { false }
+ let(:doctor) { described_class.new(logger, model_names: model_names, token_names: token_names, dry_run: dry_run) }
+
+ let_it_be(:functional_project) { create(:project).tap(&:runners_token) }
+ let_it_be(:functional_group) { create(:group).tap(&:runners_token) }
+
+ let(:broken_project) { create(:project).tap { |project| project.update_columns(runners_token_encrypted: 'aaa') } }
+ let(:project_with_cipher_error) do
+ create(:project).tap do |project|
+ project.update_columns(
+ runners_token_encrypted: '|rXs75DSHXPE9MGAIgyxcut8pZc72gaa/2ojU0GS1+R+cXNqkbUB13Vb5BaMwf47d98980fc1')
+ end
+ end
+
+ let(:broken_group) { create(:group, runners_token_encrypted: 'aaa') }
+
+ subject(:run!) do
+ expect(logger).to receive(:info).with(
+ "Resetting #{token_names.join(', ')} on #{model_names.join(', ')} if they can not be read"
+ )
+ expect(logger).to receive(:info).with('Done!')
+ doctor.run!
+ end
+
+ before do
+ allow(logger).to receive(:info).with(%r{Checked \d/\d Projects})
+ allow(logger).to receive(:info).with(%r{Checked \d Projects})
+ allow(logger).to receive(:info).with(%r{Checked \d/\d Groups})
+ allow(logger).to receive(:info).with(%r{Checked \d Groups})
+ end
+
+ it 'fixes broken project and not the functional project' do
+ expect(logger).to receive(:debug).with("> Fix Project[#{broken_project.id}].runners_token")
+
+ expect { run! }.to change { broken_project.reload.runners_token_encrypted }.from('aaa')
+ .and not_change { functional_project.reload.runners_token_encrypted }
+ expect { broken_project.runners_token }.not_to raise_error
+ end
+
+ it 'fixes project with cipher error' do
+ expect { project_with_cipher_error.runners_token }.to raise_error(OpenSSL::Cipher::CipherError)
+ expect(logger).to receive(:debug).with("> Fix Project[#{project_with_cipher_error.id}].runners_token")
+
+ expect { run! }.to change { project_with_cipher_error.reload.runners_token_encrypted }
+ expect { project_with_cipher_error.runners_token }.not_to raise_error
+ end
+
+ it 'fixes broken group and not the functional group' do
+ expect(logger).to receive(:debug).with("> Fix Group[#{broken_group.id}].runners_token")
+
+ expect { run! }.to change { broken_group.reload.runners_token_encrypted }.from('aaa')
+ .and not_change { functional_group.reload.runners_token_encrypted }
+
+ expect { broken_group.runners_token }.not_to raise_error
+ end
+
+ context 'when one model specified' do
+ let(:model_names) { %w[Project] }
+
+ it 'fixes broken project' do
+ expect(logger).to receive(:debug).with("> Fix Project[#{broken_project.id}].runners_token")
+
+ expect { run! }.to change { broken_project.reload.runners_token_encrypted }.from('aaa')
+ expect { broken_project.runners_token }.not_to raise_error
+ end
+
+ it 'does not fix other models' do
+ expect { run! }.not_to change { broken_group.reload.runners_token_encrypted }.from('aaa')
+ end
+ end
+
+ context 'when non-existing token field is given' do
+ let(:token_names) { %w[nonexisting_token] }
+
+ it 'does not fix anything' do
+ expect { run! }.not_to change { broken_project.reload.runners_token_encrypted }.from('aaa')
+ end
+ end
+
+ context 'when executing in a dry-run mode' do
+ let(:dry_run) { true }
+
+ it 'prints info about fixed project, but does not actually do anything' do
+ expect(logger).to receive(:info).with('Executing in DRY RUN mode, no records will actually be updated')
+ expect(logger).to receive(:debug).with("> Fix Project[#{broken_project.id}].runners_token")
+
+ expect { run! }.not_to change { broken_project.reload.runners_token_encrypted }.from('aaa')
+ expect { broken_project.runners_token }.to raise_error(TypeError)
+ end
+ end
+
+ it 'prints progress along the way' do
+ stub_const('Gitlab::Doctor::ResetTokens::PRINT_PROGRESS_EVERY', 1)
+
+ broken_project
+ project_with_cipher_error
+
+ expect(logger).to receive(:info).with(
+ "Resetting #{token_names.join(', ')} on #{model_names.join(', ')} if they can not be read"
+ )
+ expect(logger).to receive(:info).with('Checked 1/3 Projects')
+ expect(logger).to receive(:debug).with("> Fix Project[#{broken_project.id}].runners_token")
+ expect(logger).to receive(:info).with('Checked 2/3 Projects')
+ expect(logger).to receive(:debug).with("> Fix Project[#{project_with_cipher_error.id}].runners_token")
+ expect(logger).to receive(:info).with('Checked 3/3 Projects')
+ expect(logger).to receive(:info).with('Done!')
+
+ doctor.run!
+ end
+
+ it "prints 'Something went wrong' error when encounters unexpected exception, but continues" do
+ broken_project
+ project_with_cipher_error
+
+ expect(logger).to receive(:debug).with(
+ "> Something went wrong for Project[#{broken_project.id}].runners_token: Error message")
+ expect(logger).to receive(:debug).with("> Fix Project[#{project_with_cipher_error.id}].runners_token")
+
+ expect(broken_project).to receive(:runners_token).and_raise("Error message")
+ expect(Project).to receive(:find_each).and_return([broken_project, project_with_cipher_error].each)
+
+ expect { run! }.to not_change { broken_project.reload.runners_token_encrypted }.from('aaa')
+ .and change { project_with_cipher_error.reload.runners_token_encrypted }
+ end
+end
diff --git a/spec/tasks/gitlab/doctor/secrets_rake_spec.rb b/spec/tasks/gitlab/doctor/secrets_rake_spec.rb
new file mode 100644
index 0000000000000..22ecd91e1a0be
--- /dev/null
+++ b/spec/tasks/gitlab/doctor/secrets_rake_spec.rb
@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'rake_helper'
+
+RSpec.describe 'gitlab:doctor:reset_encrypted_tokens', :silence_stdout, feature_category: :runner_fleet do
+ let(:model_names) { 'Project,Group' }
+ let(:token_names) { 'runners_token' }
+
+ let(:project_with_cipher_error) do
+ create(:project).tap do |project|
+ project.update_columns(runners_token_encrypted:
+ '|rXs75DSHXPE9MGAIgyxcut8pZc72gaa/2ojU0GS1+R+cXNqkbUB13Vb5BaMwf47d98980fc1')
+ end
+ end
+
+ before(:context) do
+ Rake.application.rake_require 'tasks/gitlab/doctor/secrets'
+ end
+
+ before do
+ stub_env('MODEL_NAMES', model_names)
+ stub_env('TOKEN_NAMES', token_names)
+ end
+
+ subject(:run!) do
+ run_rake_task('gitlab:doctor:reset_encrypted_tokens')
+ end
+
+ it 'properly parces parameters from the environment variables' do
+ expect_next_instance_of(::Gitlab::Doctor::ResetTokens, anything,
+ model_names: %w[Project Group],
+ token_names: %w[runners_token],
+ dry_run: true) do |service|
+ expect(service).to receive(:run!).and_call_original
+ end
+
+ run!
+ end
+
+ it "doesn't do anything in DRY_RUN mode(default)" do
+ expect do
+ run!
+ end.not_to change { project_with_cipher_error.reload.runners_token_encrypted }
+ end
+
+ it 'regenerates broken token if DRY_RUN is set to false' do
+ stub_env('DRY_RUN', false)
+
+ expect { project_with_cipher_error.runners_token }.to raise_error(OpenSSL::Cipher::CipherError)
+ expect do
+ run!
+ end.to change { project_with_cipher_error.reload.runners_token_encrypted }
+
+ expect { project_with_cipher_error.runners_token }.not_to raise_error
+ end
+end
--
GitLab