Add rake task for resetting unreadable encrypted tokens

We have runners_tokens in DB that can not be decrypted.
Automatically regenerating them is dangerous,
so we provide admins with the rake task to do that.

lib/gitlab/doctor/reset_tokens.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Doctor
+    class ResetTokens
+      attr_reader :logger
+
+      PRINT_PROGRESS_EVERY = 1000
+
+      def initialize(logger, model_names:, token_names:, dry_run: true)
+        @logger = logger
+        @model_names = model_names
+        @token_names = token_names
+        @dry_run = dry_run
+      end
+
+      def run!
+        logger.info "Resetting #{@token_names.join(', ')} on #{@model_names.join(', ')} if they can not be read"
+        logger.info "Executing in DRY RUN mode, no records will actually be updated" if @dry_run
+        Rails.application.eager_load!
+
+        models_with_encrypted_tokens.each do |model|
+          fix_model(model)
+        end
+        logger.info "Done!"
+      end
+
+      private
+
+      def fix_model(model)
+        matched_token_names = @token_names & model.encrypted_token_authenticatable_fields.map(&:to_s)
+
+        return if matched_token_names.empty?
+
+        total_count = model.count
+
+        model.find_each.with_index do |instance, index|
+          matched_token_names.each do |attribute_name|
+            fix_attribute(instance, attribute_name)
+          end
+
+          logger.info "Checked #{index + 1}/#{total_count} #{model.name.pluralize}" if index % PRINT_PROGRESS_EVERY == 0
+        end
+        logger.info "Checked #{total_count} #{model.name.pluralize}"
+      end
+
+      def fix_attribute(instance, attribute_name)
+        instance.public_send(attribute_name) # rubocop:disable GitlabSecurity/PublicSend
+      rescue OpenSSL::Cipher::CipherError, TypeError
+        logger.debug "> Fix #{instance.class.name}[#{instance.id}].#{attribute_name}"
+        instance.public_send("reset_#{attribute_name}!") unless @dry_run # rubocop:disable GitlabSecurity/PublicSend
+      rescue StandardError => e
+        logger.debug(
+          "> Something went wrong for #{instance.class.name}[#{instance.id}].#{attribute_name}: #{e}".color(:red))
+
+        false
+      end
+
+      def models_with_encrypted_tokens
+        ApplicationRecord.descendants.select do |model|
+          @model_names.include?(model.name) && model.include?(TokenAuthenticatable)
+        end
+      end
+    end
+  end
+end

lib/tasks/gitlab/doctor/secrets.rake

@@ -10,5 +10,21 @@ namespace :gitlab do
 
       Gitlab::Doctor::Secrets.new(logger).run!
     end
+
+    desc "GitLab | Reset encrypted tokens for specific models"
+    task reset_encrypted_tokens: :gitlab_environment do
+      logger = Logger.new($stdout)
+
+      logger.level = Gitlab::Utils.to_boolean(ENV['VERBOSE']) ? Logger::DEBUG : Logger::INFO
+      model_names = ENV['MODEL_NAMES']&.split(',')
+      token_names = ENV['TOKEN_NAMES']&.split(',')
+      dry_run = Gitlab::Utils.to_boolean(ENV['DRY_RUN'])
+      dry_run = true if dry_run.nil?
+
+      next logger.info("No models were specified, please use MODEL_NAMES environment variable") unless model_names
+      next logger.info("No tokens were specified, please use TOKEN_NAMES environment variable") unless token_names
+
+      Gitlab::Doctor::ResetTokens.new(logger, model_names: model_names, token_names: token_names, dry_run: dry_run).run!
+    end
   end
 end

spec/lib/gitlab/doctor/reset_tokens_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Doctor::ResetTokens, feature_category: :runner_fleet do
+  let(:logger) { instance_double('Logger') }
+  let(:model_names) { %w[Project Group] }
+  let(:token_names) { %w[runners_token] }
+  let(:dry_run) { false }
+  let(:doctor) { described_class.new(logger, model_names: model_names, token_names: token_names, dry_run: dry_run) }
+
+  let_it_be(:functional_project) { create(:project).tap(&:runners_token) }
+  let_it_be(:functional_group) { create(:group).tap(&:runners_token) }
+
+  let(:broken_project) { create(:project).tap { |project| project.update_columns(runners_token_encrypted: 'aaa') } }
+  let(:project_with_cipher_error) do
+    create(:project).tap do |project|
+      project.update_columns(
+        runners_token_encrypted: '|rXs75DSHXPE9MGAIgyxcut8pZc72gaa/2ojU0GS1+R+cXNqkbUB13Vb5BaMwf47d98980fc1')
+    end
+  end
+
+  let(:broken_group) { create(:group, runners_token_encrypted: 'aaa') }
+
+  subject(:run!) do
+    expect(logger).to receive(:info).with(
+      "Resetting #{token_names.join(', ')} on #{model_names.join(', ')} if they can not be read"
+    )
+    expect(logger).to receive(:info).with('Done!')
+    doctor.run!
+  end
+
+  before do
+    allow(logger).to receive(:info).with(%r{Checked \d/\d Projects})
+    allow(logger).to receive(:info).with(%r{Checked \d Projects})
+    allow(logger).to receive(:info).with(%r{Checked \d/\d Groups})
+    allow(logger).to receive(:info).with(%r{Checked \d Groups})
+  end
+
+  it 'fixes broken project and not the functional project' do
+    expect(logger).to receive(:debug).with("> Fix Project[#{broken_project.id}].runners_token")
+
+    expect { run! }.to change { broken_project.reload.runners_token_encrypted }.from('aaa')
+      .and not_change { functional_project.reload.runners_token_encrypted }
+    expect { broken_project.runners_token }.not_to raise_error
+  end
+
+  it 'fixes project with cipher error' do
+    expect { project_with_cipher_error.runners_token }.to raise_error(OpenSSL::Cipher::CipherError)
+    expect(logger).to receive(:debug).with("> Fix Project[#{project_with_cipher_error.id}].runners_token")
+
+    expect { run! }.to change { project_with_cipher_error.reload.runners_token_encrypted }
+    expect { project_with_cipher_error.runners_token }.not_to raise_error
+  end
+
+  it 'fixes broken group and not the functional group' do
+    expect(logger).to receive(:debug).with("> Fix Group[#{broken_group.id}].runners_token")
+
+    expect { run! }.to change { broken_group.reload.runners_token_encrypted }.from('aaa')
+      .and not_change { functional_group.reload.runners_token_encrypted }
+
+    expect { broken_group.runners_token }.not_to raise_error
+  end
+
+  context 'when one model specified' do
+    let(:model_names) { %w[Project] }
+
+    it 'fixes broken project' do
+      expect(logger).to receive(:debug).with("> Fix Project[#{broken_project.id}].runners_token")
+
+      expect { run! }.to change { broken_project.reload.runners_token_encrypted }.from('aaa')
+      expect { broken_project.runners_token }.not_to raise_error
+    end
+
+    it 'does not fix other models' do
+      expect { run! }.not_to change { broken_group.reload.runners_token_encrypted }.from('aaa')
+    end
+  end
+
+  context 'when non-existing token field is given' do
+    let(:token_names) { %w[nonexisting_token] }
+
+    it 'does not fix anything' do
+      expect { run! }.not_to change { broken_project.reload.runners_token_encrypted }.from('aaa')
+    end
+  end
+
+  context 'when executing in a dry-run mode' do
+    let(:dry_run) { true }
+
+    it 'prints info about fixed project, but does not actually do anything' do
+      expect(logger).to receive(:info).with('Executing in DRY RUN mode, no records will actually be updated')
+      expect(logger).to receive(:debug).with("> Fix Project[#{broken_project.id}].runners_token")
+
+      expect { run! }.not_to change { broken_project.reload.runners_token_encrypted }.from('aaa')
+      expect { broken_project.runners_token }.to raise_error(TypeError)
+    end
+  end
+
+  it 'prints progress along the way' do
+    stub_const('Gitlab::Doctor::ResetTokens::PRINT_PROGRESS_EVERY', 1)
+
+    broken_project
+    project_with_cipher_error
+
+    expect(logger).to receive(:info).with(
+      "Resetting #{token_names.join(', ')} on #{model_names.join(', ')} if they can not be read"
+    )
+    expect(logger).to receive(:info).with('Checked 1/3 Projects')
+    expect(logger).to receive(:debug).with("> Fix Project[#{broken_project.id}].runners_token")
+    expect(logger).to receive(:info).with('Checked 2/3 Projects')
+    expect(logger).to receive(:debug).with("> Fix Project[#{project_with_cipher_error.id}].runners_token")
+    expect(logger).to receive(:info).with('Checked 3/3 Projects')
+    expect(logger).to receive(:info).with('Done!')
+
+    doctor.run!
+  end
+
+  it "prints 'Something went wrong' error when encounters unexpected exception, but continues" do
+    broken_project
+    project_with_cipher_error
+
+    expect(logger).to receive(:debug).with(
+      "> Something went wrong for Project[#{broken_project.id}].runners_token: Error message")
+    expect(logger).to receive(:debug).with("> Fix Project[#{project_with_cipher_error.id}].runners_token")
+
+    expect(broken_project).to receive(:runners_token).and_raise("Error message")
+    expect(Project).to receive(:find_each).and_return([broken_project, project_with_cipher_error].each)
+
+    expect { run! }.to not_change { broken_project.reload.runners_token_encrypted }.from('aaa')
+      .and change { project_with_cipher_error.reload.runners_token_encrypted }
+  end
+end

spec/tasks/gitlab/doctor/secrets_rake_spec.rb

+# frozen_string_literal: true
+
+require 'rake_helper'
+
+RSpec.describe 'gitlab:doctor:reset_encrypted_tokens', :silence_stdout, feature_category: :runner_fleet do
+  let(:model_names) { 'Project,Group' }
+  let(:token_names) { 'runners_token' }
+
+  let(:project_with_cipher_error) do
+    create(:project).tap do |project|
+      project.update_columns(runners_token_encrypted:
+        '|rXs75DSHXPE9MGAIgyxcut8pZc72gaa/2ojU0GS1+R+cXNqkbUB13Vb5BaMwf47d98980fc1')
+    end
+  end
+
+  before(:context) do
+    Rake.application.rake_require 'tasks/gitlab/doctor/secrets'
+  end
+
+  before do
+    stub_env('MODEL_NAMES', model_names)
+    stub_env('TOKEN_NAMES', token_names)
+  end
+
+  subject(:run!) do
+    run_rake_task('gitlab:doctor:reset_encrypted_tokens')
+  end
+
+  it 'properly parces parameters from the environment variables' do
+    expect_next_instance_of(::Gitlab::Doctor::ResetTokens, anything,
+      model_names: %w[Project Group],
+      token_names: %w[runners_token],
+      dry_run: true) do |service|
+      expect(service).to receive(:run!).and_call_original
+    end
+
+    run!
+  end
+
+  it "doesn't do anything in DRY_RUN mode(default)" do
+    expect do
+      run!
+    end.not_to change { project_with_cipher_error.reload.runners_token_encrypted }
+  end
+
+  it 'regenerates broken token if DRY_RUN is set to false' do
+    stub_env('DRY_RUN', false)
+
+    expect { project_with_cipher_error.runners_token }.to raise_error(OpenSSL::Cipher::CipherError)
+    expect do
+      run!
+    end.to change { project_with_cipher_error.reload.runners_token_encrypted }
+
+    expect { project_with_cipher_error.runners_token }.not_to raise_error
+  end
+end