Merge branch 'jp-def-answer' into 'master'

Consider unformatted answers as final answers

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/126166



Merged-by: Bojan Marjanovic <bmarjanovic@gitlab.com>
Approved-by: Bojan Marjanovic <bmarjanovic@gitlab.com>
Approved-by: Gosia Ksionek <mksionek@gitlab.com>
Reviewed-by: Bojan Marjanovic <bmarjanovic@gitlab.com>
Co-authored-by: Jan Provaznik <jprovaznik@gitlab.com>

ee/lib/gitlab/llm/chain/parsers/chain_of_thought_parser.rb

@@ -14,6 +14,9 @@ def parse
             parse_action_input
             parse_thought
             parse_final_answer
+
+            # this should be last (fallback) step after all parsing is done
+            final_answer_from_unformatted_response
           end
 
           private
@@ -49,6 +52,19 @@ def parse_final_answer
 
             @final_answer = final_answer&.strip
           end
+
+          # if response doesn't follow expected format, it usually means it's
+          # a final answer (although there is a risk of hallucination). Such
+          # response is treated as final response instead of returning "I
+          # don't know"
+          def final_answer_from_unformatted_response
+            return if action || action_input || thought || final_answer
+
+            answer = output.to_s.strip
+            return if answer.empty?
+
+            @final_answer = answer
+          end
         end
       end
     end

ee/spec/lib/gitlab/llm/chain/answer_spec.rb

@@ -51,5 +51,23 @@
         expect(answer.content).to eq(described_class.default_final_message)
       end
     end
+
+    context 'when response is empty' do
+      let(:input) { '' }
+
+      it 'returns final answer with default response' do
+        expect(answer.is_final?).to eq(true)
+        expect(answer.content).to eq(described_class.default_final_message)
+      end
+    end
+
+    context 'when tool does not contain any of expected keyword' do
+      let(:input) { 'Here is my freestyle answer.' }
+
+      it 'returns final answer with default response' do
+        expect(answer.is_final?).to eq(true)
+        expect(answer.content).to eq(input)
+      end
+    end
   end
 end

ee/spec/lib/gitlab/llm/chain/tools/json_reader/executor_spec.rb

@@ -133,7 +133,7 @@
         context 'when the response contains no action' do
           let(:ai_response) do
             <<~PROMPT
-                I'm here for the birthday party. Beep beep boop.
+                Action Input: I'm here for the birthday party. Beep beep boop.
             PROMPT
           end
 
@@ -144,6 +144,18 @@
             expect(response.content).to include(error_msg)
           end
         end
+
+        context 'when the response does not contain any keywords' do
+          let(:ai_response) do
+            <<~PROMPT
+                I'm here for the birthday party. Beep beep boop.
+            PROMPT
+          end
+
+          it 'returns final response' do
+            expect_answer_with_content(ai_response.strip)
+          end
+        end
       end
     end
   end