Introduce Gitlab::Redis::Workhorse for migration

All future workhorse-related workloads will target this module.

config/feature_flags/development/use_primary_and_secondary_stores_for_workhorse.yml

+---
+name: use_primary_and_secondary_stores_for_workhorse
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/127577
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/423420
+milestone: '16.4'
+type: development
+group: group::scalability
+default_enabled: false

config/feature_flags/development/use_primary_store_as_default_for_workhorse.yml

+---
+name: use_primary_store_as_default_for_workhorse
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/127577
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/423420
+milestone: '16.4'
+type: development
+group: group::scalability
+default_enabled: false

lib/gitlab/redis.rb

@@ -18,7 +18,8 @@ module Redis
       Gitlab::Redis::Sessions,
       Gitlab::Redis::SharedState,
       Gitlab::Redis::TraceChunks,
-      Gitlab::Redis::Chat
+      Gitlab::Redis::Chat,
+      Gitlab::Redis::Workhorse
     ].freeze
   end
 end

lib/gitlab/redis/multi_store.rb

@@ -71,6 +71,7 @@ def message
         incr
         incrby
         mapped_hmset
+        publish
         rpush
         sadd
         sadd?

lib/gitlab/redis/workhorse.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Redis
+    class Workhorse < ::Gitlab::Redis::Wrapper
+      class << self
+        def config_fallback
+          SharedState
+        end
+
+        private
+
+        def redis
+          primary_store = ::Redis.new(params)
+          secondary_store = ::Redis.new(config_fallback.params)
+
+          MultiStore.new(primary_store, secondary_store, store_name)
+        end
+      end
+    end
+  end
+end

lib/gitlab/setup_helper.rb

@@ -38,11 +38,11 @@ def configuration_toml(dir, _, _)
         end
 
         def redis_url
-          Gitlab::Redis::SharedState.url
+          Gitlab::Redis::Workhorse.url
         end
 
         def redis_db
-          Gitlab::Redis::SharedState.params.fetch(:db, 0)
+          Gitlab::Redis::Workhorse.params.fetch(:db, 0)
         end
 
         def get_config_path(dir, _)

lib/gitlab/workhorse.rb

@@ -228,7 +228,7 @@ def secret_path
       end
 
       def set_key_and_notify(key, value, expire: nil, overwrite: true)
-        Gitlab::Redis::SharedState.with do |redis|
+        with_redis do |redis|
           result = redis.set(key, value, ex: expire, nx: !overwrite)
           if result
             redis.publish(NOTIFICATION_PREFIX + key, value)
@@ -249,6 +249,15 @@ def detect_content_type
 
       protected
 
+      def with_redis(&blk)
+        if Feature.enabled?(:use_primary_and_secondary_stores_for_workhorse) ||
+            Feature.enabled?(:use_primary_store_as_default_for_workhorse)
+          Gitlab::Redis::Workhorse.with(&blk) # rubocop:disable CodeReuse/ActiveRecord
+        else
+          Gitlab::Redis::SharedState.with(&blk) # rubocop:disable CodeReuse/ActiveRecord
+        end
+      end
+
       # This is the outermost encoding of a senddata: header. It is safe for
       # inclusion in HTTP response headers
       def encode(hash)

spec/lib/gitlab/redis/workhorse_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Redis::Workhorse, feature_category: :scalability do
+  include_examples "redis_new_instance_shared_examples", 'workhorse', Gitlab::Redis::SharedState
+  include_examples "redis_shared_examples"
+
+  describe '#pool' do
+    let(:config_new_format_host) { "spec/fixtures/config/redis_new_format_host.yml" }
+    let(:config_new_format_socket) { "spec/fixtures/config/redis_new_format_socket.yml" }
+
+    subject { described_class.pool }
+
+    before do
+      allow(described_class).to receive(:config_file_name).and_return(config_new_format_host)
+
+      # Override rails root to avoid having our fixtures overwritten by `redis.yml` if it exists
+      allow(Gitlab::Redis::SharedState).to receive(:rails_root).and_return(mktmpdir)
+      allow(Gitlab::Redis::SharedState).to receive(:config_file_name).and_return(config_new_format_socket)
+    end
+
+    around do |example|
+      clear_pool
+      example.run
+    ensure
+      clear_pool
+    end
+
+    it 'instantiates an instance of MultiStore' do
+      subject.with do |redis_instance|
+        expect(redis_instance).to be_instance_of(::Gitlab::Redis::MultiStore)
+
+        expect(redis_instance.primary_store.connection[:id]).to eq("redis://test-host:6379/99")
+        expect(redis_instance.secondary_store.connection[:id]).to eq("unix:///path/to/redis.sock/0")
+
+        expect(redis_instance.instance_name).to eq('Workhorse')
+      end
+    end
+
+    it_behaves_like 'multi store feature flags', :use_primary_and_secondary_stores_for_workhorse,
+      :use_primary_store_as_default_for_workhorse
+  end
+end

spec/lib/gitlab/setup_helper/workhorse_spec.rb

@@ -24,8 +24,8 @@
   end
 
   describe '.redis_url' do
-    it 'matches the SharedState URL' do
-      expect(Gitlab::Redis::SharedState).to receive(:url).and_return('foo')
+    it 'matches the Workhorse URL' do
+      expect(Gitlab::Redis::Workhorse).to receive(:url).and_return('foo')
 
       expect(described_class.redis_url).to eq('foo')
     end
@@ -34,14 +34,14 @@
   describe '.redis_db' do
     subject { described_class.redis_db }
 
-    it 'matches the SharedState DB' do
-      expect(Gitlab::Redis::SharedState).to receive(:params).and_return(db: 1)
+    it 'matches the Workhorse DB' do
+      expect(Gitlab::Redis::Workhorse).to receive(:params).and_return(db: 1)
 
       is_expected.to eq(1)
     end
 
     it 'defaults to 0 if unspecified' do
-      expect(Gitlab::Redis::SharedState).to receive(:params).and_return({})
+      expect(Gitlab::Redis::Workhorse).to receive(:params).and_return({})
 
       is_expected.to eq(0)
     end

spec/lib/gitlab/workhorse_spec.rb

@@ -371,13 +371,13 @@ def call_verify(headers)
 
     subject { described_class.set_key_and_notify(key, value, overwrite: overwrite) }
 
-    shared_examples 'set and notify' do
+    shared_examples 'set and notify' do |redis = Gitlab::Redis::Workhorse|
       it 'set and return the same value' do
         is_expected.to eq(value)
       end
 
       it 'set and notify' do
-        expect(Gitlab::Redis::SharedState).to receive(:with).and_call_original
+        expect(redis).to receive(:with).and_call_original
         expect_any_instance_of(::Redis).to receive(:publish)
           .with(described_class::NOTIFICATION_PREFIX + 'test-key', "test-value")
 
@@ -389,6 +389,39 @@ def call_verify(headers)
       let(:overwrite) { true }
 
       it_behaves_like 'set and notify'
+
+      context 'when workhorse migration feature flags are disabled' do
+        before do
+          stub_feature_flags(
+            use_primary_and_secondary_stores_for_workhorse: false,
+            use_primary_store_as_default_for_workhorse: false
+          )
+        end
+
+        it_behaves_like 'set and notify', Gitlab::Redis::SharedState
+      end
+
+      context 'when either workhorse migration feature flags are enabled' do
+        context 'when use_primary_and_secondary_stores_for_workhorse is enabled' do
+          before do
+            stub_feature_flags(
+              use_primary_store_as_default_for_workhorse: false
+            )
+          end
+
+          it_behaves_like 'set and notify'
+        end
+
+        context 'when use_primary_store_as_default_for_workhorse is enabled' do
+          before do
+            stub_feature_flags(
+              use_primary_and_secondary_stores_for_workhorse: false
+            )
+          end
+
+          it_behaves_like 'set and notify'
+        end
+      end
     end
 
     context 'when we set an existing key' do