Merge branch '433326-bump-retries-for-stage-workers-and-advance-stage-workers' into 'master'

Increase retries for Stage & AdvanceStage workers

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/141734



Merged-by: George Koltsov <gkoltsov@gitlab.com>
Approved-by: George Koltsov <gkoltsov@gitlab.com>
Approved-by: Carla Drago <cdrago@gitlab.com>
Co-authored-by: Luke Duncalfe <lduncalfe@eml.cc>

app/workers/concerns/gitlab/bitbucket_import/stage_methods.rb

@@ -14,7 +14,7 @@ module StageMethods
 
         data_consistency :always
 
-        sidekiq_options dead: false, retry: 3
+        sidekiq_options dead: false, retry: 6
 
         sidekiq_retries_exhausted do |msg, e|
           Gitlab::Import::ImportFailureService.track(

app/workers/concerns/gitlab/bitbucket_server_import/stage_methods.rb

@@ -14,7 +14,7 @@ module StageMethods
 
         data_consistency :always
 
-        sidekiq_options dead: false, retry: 3
+        sidekiq_options dead: false, retry: 6
 
         sidekiq_retries_exhausted do |msg, e|
           Gitlab::Import::ImportFailureService.track(

app/workers/concerns/gitlab/jira_import/import_worker.rb

@@ -8,9 +8,9 @@ module ImportWorker
       included do
         include ApplicationWorker
 
-        sidekiq_options retry: 3
         include ProjectImportOptions
         include Gitlab::JiraImport::QueueOptions
+        sidekiq_options retry: 6
       end
 
       def perform(project_id)

app/workers/gitlab/bitbucket_import/advance_stage_worker.rb

@@ -12,10 +12,6 @@ class AdvanceStageWorker # rubocop:disable Scalability/IdempotentWorker
 
       data_consistency :delayed
 
-      sidekiq_options dead: false, retry: 3
-
-      feature_category :importers
-
       loggable_arguments 1, 2
 
       # The known importer stages and their corresponding Sidekiq workers.

app/workers/gitlab/bitbucket_server_import/advance_stage_worker.rb

@@ -12,10 +12,6 @@ class AdvanceStageWorker # rubocop:disable Scalability/IdempotentWorker
 
       data_consistency :delayed
 
-      sidekiq_options dead: false, retry: 3
-
-      feature_category :importers
-
       loggable_arguments 1, 2
 
       # The known importer stages and their corresponding Sidekiq workers.

app/workers/gitlab/github_import/advance_stage_worker.rb

@@ -14,8 +14,6 @@ class AdvanceStageWorker # rubocop:disable Scalability/IdempotentWorker
       include ::Gitlab::Import::AdvanceStage
 
       loggable_arguments 1, 2
-      sidekiq_options retry: 6, dead: false
-      feature_category :importers
 
       # The known importer stages and their corresponding Sidekiq workers.
       #

app/workers/gitlab/import/advance_stage.rb

@@ -3,6 +3,8 @@
 module Gitlab
   module Import
     module AdvanceStage
+      extend ActiveSupport::Concern
+
       INTERVAL = 30.seconds.to_i
       TIMEOUT_DURATION = 2.hours
 
@@ -12,6 +14,11 @@ module AdvanceStage
       # continuing to the next waiter.
       BLOCKING_WAIT_TIME = 5
 
+      included do
+        sidekiq_options dead: false, retry: 6
+        feature_category :importers
+      end
+
       # project_id - The ID of the project being imported.
       # waiters - A Hash mapping Gitlab::JobWaiter keys to the number of
       #           remaining jobs.

app/workers/gitlab/jira_import/advance_stage_worker.rb

@@ -7,7 +7,6 @@ class AdvanceStageWorker # rubocop:disable Scalability/IdempotentWorker
 
       data_consistency :always
 
-      sidekiq_options retry: 3
       include QueueOptions
       include ::Gitlab::Import::AdvanceStage
 

app/workers/gitlab/jira_import/stage/start_import_worker.rb

@@ -8,11 +8,12 @@ class StartImportWorker # rubocop:disable Scalability/IdempotentWorker
 
         data_consistency :always
 
-        sidekiq_options retry: 3
         include ProjectStartImport
         include ProjectImportOptions
         include Gitlab::JiraImport::QueueOptions
 
+        sidekiq_options retry: 6
+
         attr_reader :project
 
         def perform(project_id)

spec/workers/every_sidekiq_worker_spec.rb

@@ -247,21 +247,21 @@
         'Geo::VerificationStateBackfillWorker' => false,
         'Geo::VerificationTimeoutWorker' => false,
         'Geo::VerificationWorker' => 3,
-        'Gitlab::BitbucketImport::AdvanceStageWorker' => 3,
-        'Gitlab::BitbucketImport::Stage::FinishImportWorker' => 3,
-        'Gitlab::BitbucketImport::Stage::ImportIssuesWorker' => 3,
-        'Gitlab::BitbucketImport::Stage::ImportIssuesNotesWorker' => 3,
-        'Gitlab::BitbucketImport::Stage::ImportLfsObjectsWorker' => 3,
-        'Gitlab::BitbucketImport::Stage::ImportPullRequestsWorker' => 3,
-        'Gitlab::BitbucketImport::Stage::ImportPullRequestsNotesWorker' => 3,
-        'Gitlab::BitbucketImport::Stage::ImportRepositoryWorker' => 3,
-        'Gitlab::BitbucketServerImport::AdvanceStageWorker' => 3,
-        'Gitlab::BitbucketServerImport::Stage::FinishImportWorker' => 3,
-        'Gitlab::BitbucketServerImport::Stage::ImportLfsObjectsWorker' => 3,
-        'Gitlab::BitbucketServerImport::Stage::ImportNotesWorker' => 3,
-        'Gitlab::BitbucketServerImport::Stage::ImportPullRequestsWorker' => 3,
-        'Gitlab::BitbucketServerImport::Stage::ImportRepositoryWorker' => 3,
-        'Gitlab::BitbucketServerImport::Stage::ImportUsersWorker' => 3,
+        'Gitlab::BitbucketImport::AdvanceStageWorker' => 6,
+        'Gitlab::BitbucketImport::Stage::FinishImportWorker' => 6,
+        'Gitlab::BitbucketImport::Stage::ImportIssuesWorker' => 6,
+        'Gitlab::BitbucketImport::Stage::ImportIssuesNotesWorker' => 6,
+        'Gitlab::BitbucketImport::Stage::ImportLfsObjectsWorker' => 6,
+        'Gitlab::BitbucketImport::Stage::ImportPullRequestsWorker' => 6,
+        'Gitlab::BitbucketImport::Stage::ImportPullRequestsNotesWorker' => 6,
+        'Gitlab::BitbucketImport::Stage::ImportRepositoryWorker' => 6,
+        'Gitlab::BitbucketServerImport::AdvanceStageWorker' => 6,
+        'Gitlab::BitbucketServerImport::Stage::FinishImportWorker' => 6,
+        'Gitlab::BitbucketServerImport::Stage::ImportLfsObjectsWorker' => 6,
+        'Gitlab::BitbucketServerImport::Stage::ImportNotesWorker' => 6,
+        'Gitlab::BitbucketServerImport::Stage::ImportPullRequestsWorker' => 6,
+        'Gitlab::BitbucketServerImport::Stage::ImportRepositoryWorker' => 6,
+        'Gitlab::BitbucketServerImport::Stage::ImportUsersWorker' => 6,
         'Gitlab::GithubImport::AdvanceStageWorker' => 6,
         'Gitlab::GithubImport::Attachments::ImportReleaseWorker' => 5,
         'Gitlab::GithubImport::Attachments::ImportNoteWorker' => 5,
@@ -297,14 +297,14 @@
         'Gitlab::GithubGistsImport::ImportGistWorker' => 5,
         'Gitlab::GithubGistsImport::StartImportWorker' => 5,
         'Gitlab::GithubGistsImport::FinishImportWorker' => 5,
-        'Gitlab::JiraImport::AdvanceStageWorker' => 5,
+        'Gitlab::JiraImport::AdvanceStageWorker' => 6,
         'Gitlab::JiraImport::ImportIssueWorker' => 5,
-        'Gitlab::JiraImport::Stage::FinishImportWorker' => 5,
-        'Gitlab::JiraImport::Stage::ImportAttachmentsWorker' => 5,
-        'Gitlab::JiraImport::Stage::ImportIssuesWorker' => 5,
-        'Gitlab::JiraImport::Stage::ImportLabelsWorker' => 5,
-        'Gitlab::JiraImport::Stage::ImportNotesWorker' => 5,
-        'Gitlab::JiraImport::Stage::StartImportWorker' => 5,
+        'Gitlab::JiraImport::Stage::FinishImportWorker' => 6,
+        'Gitlab::JiraImport::Stage::ImportAttachmentsWorker' => 6,
+        'Gitlab::JiraImport::Stage::ImportIssuesWorker' => 6,
+        'Gitlab::JiraImport::Stage::ImportLabelsWorker' => 6,
+        'Gitlab::JiraImport::Stage::ImportNotesWorker' => 6,
+        'Gitlab::JiraImport::Stage::StartImportWorker' => 6,
         'GitlabPerformanceBarStatsWorker' => 3,
         'GitlabSubscriptions::RefreshSeatsWorker' => 0,
         'GitlabSubscriptions::AddOnPurchases::BulkRefreshUserAssignmentsWorker' => 0,