Add deferred_count field on deferring Sidekiq jobs

This adds more observability information on how many times a job has
been deferred.

Changelog: added

lib/gitlab/sidekiq_logging/structured_logger.rb

@@ -87,6 +87,7 @@ def log_job_done(job, started_time, payload, job_exception = nil)
         payload['message'] = "#{message}: #{job_status}: #{payload['duration_s']} sec"
         payload['job_status'] = job_status
         payload['job_deferred_by'] = job['deferred_by'] if job['deferred']
+        payload['deferred_count'] = job['deferred_count'] if job['deferred']
 
         Gitlab::ExceptionLogFormatter.format!(job_exception, payload) if job_exception
 

lib/gitlab/sidekiq_middleware/defer_jobs.rb

@@ -22,6 +22,8 @@ def call(worker, job, _queue)
           # Referred in job_logger's 'log_job_done' method to compute proper 'job_status'
           job['deferred'] = true
           job['deferred_by'] = deferred_by
+          job['deferred_count'] ||= 0
+          job['deferred_count'] += 1
 
           worker.class.perform_in(delay, *job['args'])
           counter.increment({ worker: worker.class.name })

spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb

@@ -436,6 +436,7 @@
           call_subject(job, 'test_queue') do
             job['deferred'] = true
             job['deferred_by'] = :feature_flag
+            job['deferred_count'] = 1
           end
         end
       end

spec/lib/gitlab/sidekiq_middleware/defer_jobs_spec.rb

@@ -43,6 +43,13 @@ def self.name
             expect(TestDeferredWorker).to receive(:perform_in).with(described_class::DELAY, *job['args'])
             expect { |b| subject.call(TestDeferredWorker.new, job, queue, &b) }.not_to yield_control
           end
+
+          it 'increments the defer_count' do
+            (1..5).each do |count|
+              subject.call(TestDeferredWorker.new, job, queue)
+              expect(job).to include('deferred_count' => count)
+            end
+          end
         end
 
         context 'for other workers' do
@@ -51,7 +58,7 @@ def self.name
           end
         end
 
-        it 'increments the counter' do
+        it 'increments the metric counter' do
           subject.call(TestDeferredWorker.new, job, queue)
 
           counter = ::Gitlab::Metrics.registry.get(:sidekiq_jobs_deferred_total)

spec/support/shared_contexts/lib/gitlab/sidekiq_logging/structured_logger_shared_context.rb

@@ -69,7 +69,8 @@
     end_payload.merge(
       'message' => 'TestWorker JID-da883554ee4fe414012f5f42: deferred: 0.0 sec',
       'job_status' => 'deferred',
-      'job_deferred_by' => :feature_flag
+      'job_deferred_by' => :feature_flag,
+      'deferred_count' => 1
     )
   end