From d5d91112ad701855c64f1b9cfc63862fb05022dc Mon Sep 17 00:00:00 2001
From: maddievn <mvanniekerk@gitlab.com>
Date: Mon, 17 Feb 2025 15:04:22 +0200
Subject: [PATCH] ActiveContext add preprocessors for references
Changelog: added
---
.../lib/active_context/bulk_process_queue.rb | 2 +-
.../active_context/concerns/preprocessor.rb | 24 +++++++++++++++++++
.../lib/active_context/reference.rb | 20 ++++------------
.../active_context/bulk_process_queue_spec.rb | 5 ++--
4 files changed, 32 insertions(+), 19 deletions(-)
create mode 100644 gems/gitlab-active-context/lib/active_context/concerns/preprocessor.rb
diff --git a/gems/gitlab-active-context/lib/active_context/bulk_process_queue.rb b/gems/gitlab-active-context/lib/active_context/bulk_process_queue.rb
index 293420e746015..ba9737bd017d9 100644
--- a/gems/gitlab-active-context/lib/active_context/bulk_process_queue.rb
+++ b/gems/gitlab-active-context/lib/active_context/bulk_process_queue.rb
@@ -48,7 +48,7 @@ def process(redis)
refs = deserialize_all(specs_buffer)
- Reference.preload(refs).each do |ref| # rubocop: disable Rails/FindEach -- not ActiveRecord
+ Reference.preprocess_references(refs).each do |ref|
bulk_processor.process(ref)
end
diff --git a/gems/gitlab-active-context/lib/active_context/concerns/preprocessor.rb b/gems/gitlab-active-context/lib/active_context/concerns/preprocessor.rb
new file mode 100644
index 0000000000000..684c424791cff
--- /dev/null
+++ b/gems/gitlab-active-context/lib/active_context/concerns/preprocessor.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module ActiveContext
+ module Concerns
+ module Preprocessor
+ def preprocessors
+ @preprocessors ||= []
+ end
+
+ def add_preprocessor(name, &block)
+ preprocessors << { name: name, block: block }
+ end
+
+ def preprocess(refs)
+ refs_by_class = refs.group_by(&:class)
+ refs_by_class.flat_map do |klass, class_refs|
+ klass.preprocessors.reduce(class_refs) do |processed_refs, preprocessor|
+ preprocessor[:block].call(processed_refs)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-active-context/lib/active_context/reference.rb b/gems/gitlab-active-context/lib/active_context/reference.rb
index 56a7d933d3bb1..bda7d8612354d 100644
--- a/gems/gitlab-active-context/lib/active_context/reference.rb
+++ b/gems/gitlab-active-context/lib/active_context/reference.rb
@@ -3,9 +3,9 @@
module ActiveContext
class Reference
extend Concerns::ReferenceUtils
+ extend Concerns::Preprocessor
DELIMITER = '|'
- PRELOAD_BATCH_SIZE = 1_000
class << self
def deserialize(string)
@@ -22,27 +22,17 @@ def instantiate(string)
new(*deserialize_string(string))
end
- def preload(refs)
- refs.group_by(&:class).each do |klass, class_refs|
- class_refs.each_slice(PRELOAD_BATCH_SIZE) do |group_slice|
- klass.preload_refs(group_slice)
- end
- end
-
- refs
- end
-
def serialize
raise NotImplementedError
end
- def preload_refs(refs)
- refs
- end
-
def klass
name.demodulize
end
+
+ def preprocess_references(refs)
+ preprocess(refs)
+ end
end
def klass
diff --git a/gems/gitlab-active-context/spec/lib/active_context/bulk_process_queue_spec.rb b/gems/gitlab-active-context/spec/lib/active_context/bulk_process_queue_spec.rb
index c8ca03336ff58..3bddcf7b6b080 100644
--- a/gems/gitlab-active-context/spec/lib/active_context/bulk_process_queue_spec.rb
+++ b/gems/gitlab-active-context/spec/lib/active_context/bulk_process_queue_spec.rb
@@ -19,7 +19,7 @@
describe '#process' do
let(:specs) { [['spec1', 1], ['spec2', 2]] }
- let(:reference_class) { class_double("ActiveContext::Reference", preload_refs: nil).as_stubbed_const }
+ let(:reference_class) { class_double("ActiveContext::Reference").as_stubbed_const }
let(:references) { [instance_double('ActiveContext::Reference'), instance_double('ActiveContext::Reference')] }
before do
@@ -29,8 +29,7 @@
allow(bulk_process_queue).to receive(:deserialize_all).and_return(references)
allow(redis).to receive(:zremrangebyscore)
allow(references).to receive(:group_by).and_return({ reference_class => references })
- allow(reference_class).to receive(:preload_refs)
- allow(ActiveContext::Reference).to receive(:preload).and_return(references)
+ allow(reference_class).to receive(:preprocess_references).and_return(references)
end
it 'processes specs and flushes the bulk processor' do
--
GitLab