diff --git a/ee/lib/gitlab/llm/chain/parsers/chain_of_thought_parser.rb b/ee/lib/gitlab/llm/chain/parsers/chain_of_thought_parser.rb index 25580f4d7c25c30cfacff07cde3f4eed62068afb..ca978dda97783686bc548d00a6429200a2076ad4 100644 --- a/ee/lib/gitlab/llm/chain/parsers/chain_of_thought_parser.rb +++ b/ee/lib/gitlab/llm/chain/parsers/chain_of_thought_parser.rb @@ -14,6 +14,9 @@ def parse parse_action_input parse_thought parse_final_answer + + # this should be last (fallback) step after all parsing is done + final_answer_from_unformatted_response end private @@ -49,6 +52,19 @@ def parse_final_answer @final_answer = final_answer&.strip end + + # if response doesn't follow expected format, it usually means it's + # a final answer (although there is a risk of hallucination). Such + # response is treated as final response instead of returning "I + # don't know" + def final_answer_from_unformatted_response + return if action || action_input || thought || final_answer + + answer = output.to_s.strip + return if answer.empty? + + @final_answer = answer + end end end end diff --git a/ee/spec/lib/gitlab/llm/chain/answer_spec.rb b/ee/spec/lib/gitlab/llm/chain/answer_spec.rb index 488343f59f6486db8baeb2d7bf6b13092c89a554..621a4dbfb344f7486f9ffe561046623540f6b8a5 100644 --- a/ee/spec/lib/gitlab/llm/chain/answer_spec.rb +++ b/ee/spec/lib/gitlab/llm/chain/answer_spec.rb @@ -51,5 +51,23 @@ expect(answer.content).to eq(described_class.default_final_message) end end + + context 'when response is empty' do + let(:input) { '' } + + it 'returns final answer with default response' do + expect(answer.is_final?).to eq(true) + expect(answer.content).to eq(described_class.default_final_message) + end + end + + context 'when tool does not contain any of expected keyword' do + let(:input) { 'Here is my freestyle answer.' } + + it 'returns final answer with default response' do + expect(answer.is_final?).to eq(true) + expect(answer.content).to eq(input) + end + end end end diff --git a/ee/spec/lib/gitlab/llm/chain/tools/json_reader/executor_spec.rb b/ee/spec/lib/gitlab/llm/chain/tools/json_reader/executor_spec.rb index b834c4add30aaf9406cbb6a0653260d0b7f92936..9971b3e15a80e1c01a9c4f313128b9216c10e334 100644 --- a/ee/spec/lib/gitlab/llm/chain/tools/json_reader/executor_spec.rb +++ b/ee/spec/lib/gitlab/llm/chain/tools/json_reader/executor_spec.rb @@ -133,7 +133,7 @@ context 'when the response contains no action' do let(:ai_response) do <<~PROMPT - I'm here for the birthday party. Beep beep boop. + Action Input: I'm here for the birthday party. Beep beep boop. PROMPT end @@ -144,6 +144,18 @@ expect(response.content).to include(error_msg) end end + + context 'when the response does not contain any keywords' do + let(:ai_response) do + <<~PROMPT + I'm here for the birthday party. Beep beep boop. + PROMPT + end + + it 'returns final response' do + expect_answer_with_content(ai_response.strip) + end + end end end end