From aeb47d1fb039c758ba36ffe4d0a2f93b1a149076 Mon Sep 17 00:00:00 2001
From: Shubham Kumar <shubhamkrai123@gmail.com>
Date: Fri, 16 Feb 2024 03:53:53 +0000
Subject: [PATCH] Fix bitbucket spread cal job
Changelog: changed
---
.../bitbucket_import/parallel_scheduling.rb | 9 ++++-
.../parallel_scheduling.rb | 9 ++++-
.../importers/lfs_objects_importer_spec.rb | 3 +-
.../parallel_scheduling_spec.rb | 37 +++++++++++++++++++
.../importers/lfs_objects_importer_spec.rb | 3 +-
.../parallel_scheduling_spec.rb | 37 +++++++++++++++++++
6 files changed, 92 insertions(+), 6 deletions(-)
create mode 100644 spec/lib/gitlab/bitbucket_import/parallel_scheduling_spec.rb
create mode 100644 spec/lib/gitlab/bitbucket_server_import/parallel_scheduling_spec.rb
diff --git a/lib/gitlab/bitbucket_import/parallel_scheduling.rb b/lib/gitlab/bitbucket_import/parallel_scheduling.rb
index ca2597ea5cff2..09d1eb683195b 100644
--- a/lib/gitlab/bitbucket_import/parallel_scheduling.rb
+++ b/lib/gitlab/bitbucket_import/parallel_scheduling.rb
@@ -76,9 +76,14 @@ def mark_as_enqueued(object)
end
def calculate_job_delay(job_index)
- multiplier = (job_index / BATCH_SIZE)
+ runtime = Time.current - job_started_at
+ multiplier = (job_index / BATCH_SIZE.to_f)
- (multiplier * 1.minute) + 1.second
+ (multiplier * 1.minute) + 1.second - runtime
+ end
+
+ def job_started_at
+ @job_started_at ||= Time.current
end
end
end
diff --git a/lib/gitlab/bitbucket_server_import/parallel_scheduling.rb b/lib/gitlab/bitbucket_server_import/parallel_scheduling.rb
index 7fd61bbb20663..503dfc37ac221 100644
--- a/lib/gitlab/bitbucket_server_import/parallel_scheduling.rb
+++ b/lib/gitlab/bitbucket_server_import/parallel_scheduling.rb
@@ -83,9 +83,14 @@ def mark_as_processed(object)
end
def calculate_job_delay(job_index)
- multiplier = (job_index / BATCH_SIZE)
+ runtime = Time.current - job_started_at
+ multiplier = (job_index / BATCH_SIZE.to_f)
- (multiplier * 1.minute) + 1.second
+ (multiplier * 1.minute) + 1.second - runtime
+ end
+
+ def job_started_at
+ @job_started_at ||= Time.current
end
def track_import_failure!(project, exception:, **args)
diff --git a/spec/lib/gitlab/bitbucket_import/importers/lfs_objects_importer_spec.rb b/spec/lib/gitlab/bitbucket_import/importers/lfs_objects_importer_spec.rb
index fbce833726440..289a6de1e9164 100644
--- a/spec/lib/gitlab/bitbucket_import/importers/lfs_objects_importer_spec.rb
+++ b/spec/lib/gitlab/bitbucket_import/importers/lfs_objects_importer_spec.rb
@@ -46,7 +46,8 @@
end
expect(Gitlab::BitbucketImport::ImportLfsObjectWorker).to receive(:perform_in)
- .with(1.second, project.id, lfs_attributes.stringify_keys, start_with(Gitlab::JobWaiter::KEY_PREFIX))
+ .with(an_instance_of(Float), project.id,
+ lfs_attributes.stringify_keys, start_with(Gitlab::JobWaiter::KEY_PREFIX))
waiter = importer.execute
diff --git a/spec/lib/gitlab/bitbucket_import/parallel_scheduling_spec.rb b/spec/lib/gitlab/bitbucket_import/parallel_scheduling_spec.rb
new file mode 100644
index 0000000000000..7aedc8e1466c5
--- /dev/null
+++ b/spec/lib/gitlab/bitbucket_import/parallel_scheduling_spec.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BitbucketImport::ParallelScheduling, feature_category: :importers do
+ let_it_be(:project) { build(:project) }
+
+ describe '#calculate_job_delay' do
+ let(:importer_class) do
+ Class.new do
+ include Gitlab::BitbucketImport::ParallelScheduling
+
+ def collection_method
+ :issues
+ end
+ end
+ end
+
+ let(:importer) { importer_class.new(project) }
+
+ before do
+ stub_const("#{described_class}::BATCH_SIZE", 2)
+ end
+
+ it 'returns an incremental delay', :freeze_time do
+ expect(importer.send(:calculate_job_delay, 1)).to eq(0.5.minutes + 1.second)
+ expect(importer.send(:calculate_job_delay, 100)).to eq(50.minutes + 1.second)
+ end
+
+ it 'deducts the runtime from the delay', :freeze_time do
+ allow(importer).to receive(:job_started_at).and_return(1.second.ago)
+
+ expect(importer.send(:calculate_job_delay, 1)).to eq(0.5.minutes)
+ expect(importer.send(:calculate_job_delay, 100)).to eq(50.minutes)
+ end
+ end
+end
diff --git a/spec/lib/gitlab/bitbucket_server_import/importers/lfs_objects_importer_spec.rb b/spec/lib/gitlab/bitbucket_server_import/importers/lfs_objects_importer_spec.rb
index 0d66ad7c2ec30..aa901fe0146d2 100644
--- a/spec/lib/gitlab/bitbucket_server_import/importers/lfs_objects_importer_spec.rb
+++ b/spec/lib/gitlab/bitbucket_server_import/importers/lfs_objects_importer_spec.rb
@@ -46,7 +46,8 @@
end
expect(Gitlab::BitbucketServerImport::ImportLfsObjectWorker).to receive(:perform_in)
- .with(1.second, project.id, lfs_attributes.stringify_keys, start_with(Gitlab::JobWaiter::KEY_PREFIX))
+ .with(an_instance_of(Float), project.id,
+ lfs_attributes.stringify_keys, start_with(Gitlab::JobWaiter::KEY_PREFIX))
waiter = importer.execute
diff --git a/spec/lib/gitlab/bitbucket_server_import/parallel_scheduling_spec.rb b/spec/lib/gitlab/bitbucket_server_import/parallel_scheduling_spec.rb
new file mode 100644
index 0000000000000..bfd701a6ea46e
--- /dev/null
+++ b/spec/lib/gitlab/bitbucket_server_import/parallel_scheduling_spec.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BitbucketServerImport::ParallelScheduling, feature_category: :importers do
+ let_it_be(:project) { build(:project) }
+
+ describe '#calculate_job_delay' do
+ let(:importer_class) do
+ Class.new do
+ include Gitlab::BitbucketServerImport::ParallelScheduling
+
+ def collection_method
+ :issues
+ end
+ end
+ end
+
+ let(:importer) { importer_class.new(project) }
+
+ before do
+ stub_const("#{described_class}::BATCH_SIZE", 2)
+ end
+
+ it 'returns an incremental delay', :freeze_time do
+ expect(importer.send(:calculate_job_delay, 1)).to eq(0.5.minutes + 1.second)
+ expect(importer.send(:calculate_job_delay, 100)).to eq(50.minutes + 1.second)
+ end
+
+ it 'deducts the runtime from the delay', :freeze_time do
+ allow(importer).to receive(:job_started_at).and_return(1.second.ago)
+
+ expect(importer.send(:calculate_job_delay, 1)).to eq(0.5.minutes)
+ expect(importer.send(:calculate_job_delay, 100)).to eq(50.minutes)
+ end
+ end
+end
--
GitLab