Add option to retry failed helm deployment

qa/gems/gitlab-cng/lib/gitlab/cng/commands/subcommands/deployment.rb

@@ -49,6 +49,10 @@ def method_added(name)
                 type: :string,
                 repeatable: true,
                 aliases: "-e"
+              option :retry,
+                desc: "Max number of retries for failed deployment",
+                default: 0,
+                type: :numeric
 
               super(name)
             end
@@ -120,7 +124,7 @@ def kind(name = DEFAULT_HELM_RELEASE_NAME)
           def installation(name, configuration)
             Cng::Deployment::Installation.new(
               name, configuration: configuration,
-              **symbolized_options.slice(:namespace, :set, :ci, :gitlab_domain, :timeout, :chart_sha, :env)
+              **symbolized_options.slice(:namespace, :set, :ci, :gitlab_domain, :timeout, :chart_sha, :env, :retry)
             )
           end
 

qa/gems/gitlab-cng/lib/gitlab/cng/lib/deployment/installation.rb

@@ -56,6 +56,8 @@ def initialize(name, configuration:, namespace:, ci:, gitlab_domain:, timeout:,
           @set = args[:set] || []
           @extra_env = args[:env] || []
           @chart_sha = args[:chart_sha]
+          @retry_attempts = args[:retry] || 0
+          @deployment_attempts = 0
         end
 
         # Perform deployment with all the additional setup
@@ -88,7 +90,8 @@ def component_version_values
           :gitlab_domain,
           :timeout,
           :chart_sha,
-          :extra_env
+          :extra_env,
+          :retry_attempts
 
         alias_method :cli_values, :set
 
@@ -186,7 +189,12 @@ def run_deploy(chart_reference)
           Helpers::Spinner.spin("running helm deployment") do
             helm.upgrade(name, chart_reference, namespace: namespace, timeout: timeout, values: values, args: args)
           rescue Helm::Client::Error => e
-            handle_deploy_failure(e)
+            @deployment_attempts += 1
+            handle_deploy_failure(e) if @deployment_attempts > retry_attempts
+
+            log("Deployment failed, retrying...", :warn)
+            log("Error: #{e}", :warn)
+            retry
           end
           log("Deployment successful and app is available via: #{configuration.gitlab_url}", :success, bright: true)
         end
@@ -226,7 +234,7 @@ def create_license
         # @param [StandardError] error
         # @return [void]
         def handle_deploy_failure(error)
-          log("Helm upgrade failed!", :error)
+          log("Helm deployment failed!", :error)
           log("For more information on troubleshooting failures, see: '#{TROUBLESHOOTING_LINK}'", :warn)
 
           events = get_warning_events

qa/gems/gitlab-cng/spec/unit/gitlab/cng/commands/subcommands/deployment_spec.rb

@@ -46,7 +46,8 @@
         namespace: "gitlab",
         ci: false,
         gitlab_domain: "127.0.0.1.nip.io",
-        timeout: "10m"
+        timeout: "10m",
+        retry: 0
       )
       expect(Gitlab::Cng::Deployment::Configurations::Kind).to have_received(:new).with(
         namespace: "gitlab",

qa/gems/gitlab-cng/spec/unit/gitlab/cng/deployment/installation_spec.rb

@@ -11,7 +11,8 @@
         gitlab_domain: gitlab_domain,
         timeout: "10m",
         chart_sha: chart_sha,
-        env: ["RAILS_ENV_VAR=val"]
+        env: ["RAILS_ENV_VAR=val"],
+        retry: retry_attempts
       )
     end
 
@@ -20,6 +21,7 @@
     let(:chart_sha) { nil }
     let(:chart_reference) { "chart-reference" }
     let(:ci) { false }
+    let(:retry_attempts) { 0 }
 
     let(:kubeclient) do
       instance_double(Gitlab::Cng::Kubectl::Client, create_namespace: "", create_resource: "", execute: "")
@@ -100,12 +102,23 @@
         EVENTS
       end
 
-      it "automatically prints warning events" do
-        expect { expect { installation.create }.to raise_error(SystemExit) }.to output(
-          match("#{warn_event[:involvedObject][:kind]}/#{warn_event[:involvedObject][:name]}")
-          .and(match(warn_event[:message]))
-          .and(match(/For more information on troubleshooting failures, see: \S+/))
-        ).to_stdout
+      context "without retry" do
+        it "automatically prints warning events" do
+          expect { expect { installation.create }.to raise_error(SystemExit) }.to output(
+            match("#{warn_event[:involvedObject][:kind]}/#{warn_event[:involvedObject][:name]}")
+            .and(match(warn_event[:message]))
+            .and(match(/For more information on troubleshooting failures, see: \S+/))
+          ).to_stdout
+        end
+      end
+
+      context "with retry" do
+        let(:retry_attempts) { 1 }
+
+        it "retries deployment" do
+          expect { expect { installation.create }.to raise_error(SystemExit) }.to output.to_stdout
+          expect(helmclient).to have_received(:upgrade).twice
+        end
       end
     end