diff --git a/gems/gitlab-active-context/lib/active_context/bulk_process_queue.rb b/gems/gitlab-active-context/lib/active_context/bulk_process_queue.rb index 293420e74601544348174af85d17f64883ccaa6d..ba9737bd017d9266ef852947fe9e08e9ea52b805 100644 --- a/gems/gitlab-active-context/lib/active_context/bulk_process_queue.rb +++ b/gems/gitlab-active-context/lib/active_context/bulk_process_queue.rb @@ -48,7 +48,7 @@ def process(redis) refs = deserialize_all(specs_buffer) - Reference.preload(refs).each do |ref| # rubocop: disable Rails/FindEach -- not ActiveRecord + Reference.preprocess_references(refs).each do |ref| bulk_processor.process(ref) end diff --git a/gems/gitlab-active-context/lib/active_context/concerns/preprocessor.rb b/gems/gitlab-active-context/lib/active_context/concerns/preprocessor.rb new file mode 100644 index 0000000000000000000000000000000000000000..684c424791cffb12e13a1b7a240fb3f69ee28c32 --- /dev/null +++ b/gems/gitlab-active-context/lib/active_context/concerns/preprocessor.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module ActiveContext + module Concerns + module Preprocessor + def preprocessors + @preprocessors ||= [] + end + + def add_preprocessor(name, &block) + preprocessors << { name: name, block: block } + end + + def preprocess(refs) + refs_by_class = refs.group_by(&:class) + refs_by_class.flat_map do |klass, class_refs| + klass.preprocessors.reduce(class_refs) do |processed_refs, preprocessor| + preprocessor[:block].call(processed_refs) + end + end + end + end + end +end diff --git a/gems/gitlab-active-context/lib/active_context/reference.rb b/gems/gitlab-active-context/lib/active_context/reference.rb index 56a7d933d3bb10785ccbfc63ee14771543c28bf8..bda7d8612354d4d2450542cc00a6701852b96429 100644 --- a/gems/gitlab-active-context/lib/active_context/reference.rb +++ b/gems/gitlab-active-context/lib/active_context/reference.rb @@ -3,9 +3,9 @@ module ActiveContext class Reference extend Concerns::ReferenceUtils + extend Concerns::Preprocessor DELIMITER = '|' - PRELOAD_BATCH_SIZE = 1_000 class << self def deserialize(string) @@ -22,27 +22,17 @@ def instantiate(string) new(*deserialize_string(string)) end - def preload(refs) - refs.group_by(&:class).each do |klass, class_refs| - class_refs.each_slice(PRELOAD_BATCH_SIZE) do |group_slice| - klass.preload_refs(group_slice) - end - end - - refs - end - def serialize raise NotImplementedError end - def preload_refs(refs) - refs - end - def klass name.demodulize end + + def preprocess_references(refs) + preprocess(refs) + end end def klass diff --git a/gems/gitlab-active-context/spec/lib/active_context/bulk_process_queue_spec.rb b/gems/gitlab-active-context/spec/lib/active_context/bulk_process_queue_spec.rb index c8ca03336ff58b7b06487aa834fd2cb500595f81..3bddcf7b6b080229190a148990dc23c7c028bf61 100644 --- a/gems/gitlab-active-context/spec/lib/active_context/bulk_process_queue_spec.rb +++ b/gems/gitlab-active-context/spec/lib/active_context/bulk_process_queue_spec.rb @@ -19,7 +19,7 @@ describe '#process' do let(:specs) { [['spec1', 1], ['spec2', 2]] } - let(:reference_class) { class_double("ActiveContext::Reference", preload_refs: nil).as_stubbed_const } + let(:reference_class) { class_double("ActiveContext::Reference").as_stubbed_const } let(:references) { [instance_double('ActiveContext::Reference'), instance_double('ActiveContext::Reference')] } before do @@ -29,8 +29,7 @@ allow(bulk_process_queue).to receive(:deserialize_all).and_return(references) allow(redis).to receive(:zremrangebyscore) allow(references).to receive(:group_by).and_return({ reference_class => references }) - allow(reference_class).to receive(:preload_refs) - allow(ActiveContext::Reference).to receive(:preload).and_return(references) + allow(reference_class).to receive(:preprocess_references).and_return(references) end it 'processes specs and flushes the bulk processor' do