diff --git a/app/models/repository.rb b/app/models/repository.rb
index 1c02e68f2f6b1a6213f4d53c846c64fef0c5a38c..f540b00a84929ad9275f66170d6ca1b7708ab53b 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -1050,7 +1050,7 @@ def rebase_deprecated(user, merge_request)
 
     # To support the full deprecated behaviour, set the
     # `rebase_commit_sha` for the merge_request here and return the value
-    merge_request.update(rebase_commit_sha: rebase_sha)
+    merge_request.update(rebase_commit_sha: rebase_sha, merge_error: nil)
 
     rebase_sha
   end
@@ -1069,7 +1069,7 @@ def rebase(user, merge_request)
         remote_repository: merge_request.target_project.repository.raw,
         remote_branch: merge_request.target_branch
       ) do |commit_id|
-        merge_request.update!(rebase_commit_sha: commit_id)
+        merge_request.update!(rebase_commit_sha: commit_id, merge_error: nil)
       end
     end
   end
diff --git a/changelogs/unreleased/sh-fix-rebase-error-clearing.yml b/changelogs/unreleased/sh-fix-rebase-error-clearing.yml
new file mode 100644
index 0000000000000000000000000000000000000000..4f5f2779e7f0f27dfe277acd5cce86af21c17524
--- /dev/null
+++ b/changelogs/unreleased/sh-fix-rebase-error-clearing.yml
@@ -0,0 +1,5 @@
+---
+title: Properly clear the merge error upon rebase failure
+merge_request: 28319
+author:
+type: fixed
diff --git a/spec/services/merge_requests/rebase_service_spec.rb b/spec/services/merge_requests/rebase_service_spec.rb
index a443e4588d919ffac2723d51f3e95875388b93c4..0c4433453ac9a28b00c277a8e4af9d5b7f9dcaef 100644
--- a/spec/services/merge_requests/rebase_service_spec.rb
+++ b/spec/services/merge_requests/rebase_service_spec.rb
@@ -38,6 +38,32 @@
       end
     end
 
+    shared_examples 'sequence of failure and success' do
+      it 'properly clears the error message' do
+        allow(repository).to receive(:gitaly_operation_client).and_raise('Something went wrong')
+
+        service.execute(merge_request)
+
+        expect(merge_request.reload.merge_error).to eq described_class::REBASE_ERROR
+
+        allow(repository).to receive(:gitaly_operation_client).and_call_original
+
+        service.execute(merge_request)
+
+        expect(merge_request.reload.merge_error).to eq nil
+      end
+    end
+
+    it_behaves_like 'sequence of failure and success'
+
+    context 'with deprecated step rebase feature' do
+      before do
+        allow(Feature).to receive(:disabled?).with(:two_step_rebase, anything).and_return(true)
+      end
+
+      it_behaves_like 'sequence of failure and success'
+    end
+
     context 'when unexpected error occurs' do
       before do
         allow(repository).to receive(:gitaly_operation_client).and_raise('Something went wrong')