diff --git a/.rubocop_todo/style/inline_disable_annotation.yml b/.rubocop_todo/style/inline_disable_annotation.yml index 477d50e61d6d058fdceb4fc8ce054cb46c6cd46d..8e43035cd0cedff0417576738e0c2fa4df3411cf 100644 --- a/.rubocop_todo/style/inline_disable_annotation.yml +++ b/.rubocop_todo/style/inline_disable_annotation.yml @@ -1864,7 +1864,6 @@ Style/InlineDisableAnnotation: - 'ee/lib/gitlab/insights/finders/issuable_finder.rb' - 'ee/lib/gitlab/insights/reducers/count_per_period_reducer.rb' - 'ee/lib/gitlab/llm/chain/tools/identifier.rb' - - 'ee/lib/gitlab/llm/chain/tools/json_reader/executor.rb' - 'ee/lib/gitlab/llm/chain/tools/summarize_comments/executor.rb' - 'ee/lib/gitlab/llm/chat_storage.rb' - 'ee/lib/gitlab/llm/open_ai/client.rb' diff --git a/ee/lib/gitlab/llm/chain/tools/json_reader/executor.rb b/ee/lib/gitlab/llm/chain/tools/json_reader/executor.rb deleted file mode 100644 index f17a6d8acff7543439174162d644db8933684daf..0000000000000000000000000000000000000000 --- a/ee/lib/gitlab/llm/chain/tools/json_reader/executor.rb +++ /dev/null @@ -1,188 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Llm - module Chain - module Tools - module JsonReader - # this tool is currently not used. As we may come back to using it in the future, at this moment I am not - # deleting it. New code regarding serializing resources should be added to "concerns/reader_tooling" module - class Executor < Tool - include Concerns::AiDependent - - PromptRoles = ::Gitlab::Llm::Chain::Utils::Prompt - TextUtils = Gitlab::Llm::Chain::Utils::TextProcessing - - NAME = 'ResourceReader' - HUMAN_NAME = 'Prepare data' - DESCRIPTION = 'Useful tool when you need to get information about specific resource ' \ - 'that was already identified. ' \ - 'Action Input for this tools always starts with: `data`' - RESOURCE_NAME = '' - - EXAMPLE = - <<~PROMPT - Question: Who is an author of this issue - Picked tools: First: "IssueIdentifier" tool, second: "ResourceReader" tool. - Reason: You have access to the same resources as user who asks a question. - Once the resource is identified, you should use "ResourceReader" tool to fetch relevant information - about the resource. Based on this information you can present final answer. - PROMPT - - MAX_RETRIES = 3 - - PROVIDER_PROMPT_CLASSES = { - ai_gateway: ::Gitlab::Llm::Chain::Tools::JsonReader::Prompts::Anthropic, - anthropic: ::Gitlab::Llm::Chain::Tools::JsonReader::Prompts::Anthropic, - vertex_ai: ::Gitlab::Llm::Chain::Tools::JsonReader::Prompts::VertexAi - }.freeze - - def initialize(context:, options:, stream_response_handler: nil) - super - @retries = 0 - end - - def perform - begin - resource_serialized = context.resource_serialized(content_limit: provider_prompt_class::MAX_CHARACTERS) - rescue ArgumentError - return Answer.error_answer(context: context, - content: _("Unexpected error: Cannot serialize resource", resource_class: resource.class) - ) - end - - @data = Hash.from_xml(resource_serialized)['root'] - - options[:suggestions] = options[:suggestions].to_s - prompt_length = resource_serialized.length + options[:suggestions].length - - if prompt_length < provider_prompt_class::MAX_CHARACTERS - process_short_path(resource_serialized) - else - process_long_path - end - rescue StandardError => e - logger.error(message: "error message", error: e.message) - Answer.error_answer(context: context, content: _("Unexpected error")) - end - traceable :perform, run_type: 'tool' - - private - - attr_accessor :data, :retries - - def authorize - Utils::ChatAuthorizer.context(context: context).allowed? - end - - def process_short_path(resource_serialized) - content = "Please use this information about this resource: #{resource_serialized}" - - ::Gitlab::Llm::Chain::Answer.new(status: :ok, context: context, content: content, tool: nil) - end - - def process_long_path - failure_message = "#{options[:suggestions]}\nFailed to parse JSON." - return Answer.error_answer(context: context, content: failure_message) if retries >= MAX_RETRIES - - parser = Parsers::ChainOfThoughtParser.new(output: request) - parser.parse - - tool_answer = Answer.new(status: :ok, context: context, content: parser.final_answer, tool: nil) - return tool_answer if parser.final_answer - - options[:suggestions] << "\nAction: #{parser.action}\nAction Input: #{parser.action_input}\n" - options[:suggestions] << "\nThought:#{parser.thought}" unless parser.thought.blank? - - action_class = case parser.action - when /.*JsonReaderListKeys.*/ - Utils::JsonReaderListKeys - when /.*JsonReaderGetValue.*/ - Utils::JsonReaderGetValue - end - - if action_class - json_keys = action_class.handle_keys(parser.action_input, data).to_json - options[:suggestions] << "\nObservation: #{json_keys}\n" - else - self.retries += 1 - msg = "#{action_class} is not valid, Action must be either `JsonReaderListKeys` or `JsonReaderGetValue`" - options[:suggestions] << "\nObservation: #{msg}" - end - - process_long_path - rescue JSON::ParserError - # try to help out AI to fix the JSON format by adding the error as an observation - self.retries += 1 - error_message = "\nObservation: JSON has an invalid format. Please retry." - options[:suggestions] += error_message - - process_long_path - end - - # rubocop: disable Layout/LineLength - # our template - PROMPT_TEMPLATE = [ - PromptRoles.as_system( - <<~PROMPT - You are an agent designed to interact with JSON. - Your goal is to return a information relevant to make final answer as JSON. - You have access to the following tools which help you learn more about the JSON you are interacting with. - Only use the below tools. Only use the information returned by the below tools to construct your final answer. - Do not make up any information that is not contained in the JSON. - Your input to the tools should be in the form of `data['key'][0]` where `data` is the JSON blob you are interacting with, and the syntax used is Ruby. - You should only use keys that you know for a fact exist. You must validate that a key exists by seeing it previously when calling `JsonReaderListKeys`. - If you have not seen a key in one of those responses you cannot use it. - You should only add one key at a time to the path. You cannot add multiple keys at once. - If you encounter a 'KeyError', go back to the previous key, look at the available keys and try again. - - If the question does not seem to be related to the JSON, just return 'I don't know' as the answer. - Always start your interaction with the `Action: JsonReaderListKeys` tool and input 'Action Input: data' to see what keys exist in the JSON. - - Note that sometimes the value at a given path is large. In this case, you will get an error 'Value is a large dictionary, should explore its keys directly'. - In this case, you should ALWAYS follow up by using the `JsonReaderListKeys` tool to see what keys exist at that path. - Do not simply refer the user to the JSON or a section of the JSON, as this is not a valid answer. Keep digging until you find the answer and explicitly return it. - - - JsonReaderListKeys: - Can be used to list all keys at a given path. - Before calling this you should be SURE that the path to this exists. - The input is a text representation of the path in a Hash in Ruby syntax (e.g. data['key1'][0]['key2']). - - JsonReaderGetValue: - Can be used to see value in string format at a given path. - Before calling this you should be SURE that the path to this exists. - The input is a text representation of the path in a Hash in Ruby syntax (e.g. data['key1'][0]['key2']). - - - Use the following format: - - Question: the input question you must answer - Thought: you should always think about what to do - Action: the action to take, should be one from this list: [JsonReaderListKeys JsonReaderGetValue] - Action Input: the input to the action - Observation: the result of the action - ... (this Thought/Action/Action Input/Observation can repeat N times) - Thought: I now know the final answer - Final Answer: Information relevant to answering question as JSON. - REMEMBER to ALWAYS start a line with "Final Answer:" to give me the final answer. - - Begin! - PROMPT - ), - PromptRoles.as_user("Question: %<input>s"), - PromptRoles.as_assistant( - <<~PROMPT - Thought: I should look at the keys that exist in `data` to see what I have access to\n, - %<suggestions>s - - PROMPT - ) - ].freeze - end - # rubocop: enable Layout/LineLength - end - end - end - end -end diff --git a/ee/lib/gitlab/llm/chain/tools/json_reader/prompts/anthropic.rb b/ee/lib/gitlab/llm/chain/tools/json_reader/prompts/anthropic.rb deleted file mode 100644 index 20b4405d522e3c6ee5f39fc2932711e1a7abf0f3..0000000000000000000000000000000000000000 --- a/ee/lib/gitlab/llm/chain/tools/json_reader/prompts/anthropic.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Llm - module Chain - module Tools - module JsonReader - module Prompts - class Anthropic - include Concerns::AnthropicPrompt - - def self.prompt(options) - base_prompt = Utils::Prompt.no_role_text( - ::Gitlab::Llm::Chain::Tools::JsonReader::Executor::PROMPT_TEMPLATE, options - ).concat("\nThought:") - - Requests::Anthropic.prompt("\n\nHuman: #{base_prompt}\n\nAssistant:") - end - end - end - end - end - end - end -end diff --git a/ee/lib/gitlab/llm/chain/tools/json_reader/prompts/vertex_ai.rb b/ee/lib/gitlab/llm/chain/tools/json_reader/prompts/vertex_ai.rb deleted file mode 100644 index 298cfac838fa14090d5743f85b0386129028b0ad..0000000000000000000000000000000000000000 --- a/ee/lib/gitlab/llm/chain/tools/json_reader/prompts/vertex_ai.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Llm - module Chain - module Tools - module JsonReader - module Prompts - class VertexAi - include Concerns::VertexAiPrompt - - def self.prompt(options) - prompt = Utils::Prompt.no_role_text( - ::Gitlab::Llm::Chain::Tools::JsonReader::Executor::PROMPT_TEMPLATE, options - ).concat("\nThought:") - - Requests::VertexAi.prompt(prompt) - end - end - end - end - end - end - end -end diff --git a/ee/lib/gitlab/llm/chain/utils/json_reader_get_value.rb b/ee/lib/gitlab/llm/chain/utils/json_reader_get_value.rb deleted file mode 100644 index c31a628e99c58838701b6a787c755454a749f13a..0000000000000000000000000000000000000000 --- a/ee/lib/gitlab/llm/chain/utils/json_reader_get_value.rb +++ /dev/null @@ -1,16 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Llm - module Chain - module Utils - class JsonReaderGetValue < TextProcessing - def self.handle_keys(input, data) - keys = extract_keys(input) - data.dig(*keys) - end - end - end - end - end -end diff --git a/ee/lib/gitlab/llm/chain/utils/json_reader_list_keys.rb b/ee/lib/gitlab/llm/chain/utils/json_reader_list_keys.rb deleted file mode 100644 index 06e4c70ee9d6aa7badbd03bebe339ee614b7b281..0000000000000000000000000000000000000000 --- a/ee/lib/gitlab/llm/chain/utils/json_reader_list_keys.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Llm - module Chain - module Utils - class JsonReaderListKeys < TextProcessing - def self.handle_keys(input, data) - keys = extract_keys(input) - return data.keys if keys.blank? - - value = data - - keys.each do |key| - value = value[key] - unless value.respond_to?(:keys) - break "ValueError: Value at path `#{input}` is not a Hash, try to use `JsonReaderGetValue` - to get the value directly." - end - - value.keys.to_s - end - end - end - end - end - end -end diff --git a/ee/spec/lib/gitlab/llm/chain/agents/zero_shot/executor_spec.rb b/ee/spec/lib/gitlab/llm/chain/agents/zero_shot/executor_spec.rb index 8bcc6c649a42a64f8a699a7219925619fe6088b8..a1bdaf748faf8ebdb0d823b28420c6f56bf289b8 100644 --- a/ee/spec/lib/gitlab/llm/chain/agents/zero_shot/executor_spec.rb +++ b/ee/spec/lib/gitlab/llm/chain/agents/zero_shot/executor_spec.rb @@ -168,7 +168,6 @@ describe '#prompt' do let(:tools) do [ - Gitlab::Llm::Chain::Tools::JsonReader, Gitlab::Llm::Chain::Tools::IssueIdentifier, Gitlab::Llm::Chain::Tools::EpicIdentifier ] diff --git a/ee/spec/lib/gitlab/llm/chain/tools/json_reader/executor_spec.rb b/ee/spec/lib/gitlab/llm/chain/tools/json_reader/executor_spec.rb deleted file mode 100644 index c5c051a56593495864ccf72a706fb32f44bffe46..0000000000000000000000000000000000000000 --- a/ee/spec/lib/gitlab/llm/chain/tools/json_reader/executor_spec.rb +++ /dev/null @@ -1,246 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Llm::Chain::Tools::JsonReader::Executor, :aggregate_failures, :saas, feature_category: :duo_chat do - subject(:reader) { described_class.new(context: context, options: options, stream_response_handler: nil) } - - let_it_be(:user) { create(:user) } - let_it_be_with_reload(:group) { create(:group_with_plan, plan: :ultimate_plan) } - let_it_be_with_reload(:project) { create(:project, group: group, name: "My sweet project with robots in it") } - let_it_be(:issue) do - create(:issue, project: project, title: "AI should be included at birthdays", description: description_content) - end - - let_it_be(:note) { create(:note_on_issue, project: project, noteable: issue, note: note_content) } - - let(:context) do - Gitlab::Llm::Chain::GitlabContext.new( - container: project, - resource: resource, - current_user: user, - ai_request: ai_request - ) - end - - let(:ai_request) { double } - let(:resource) { issue } - let(:options) { { suggestions: +"", input: "question" } } - let(:ai_response) { "It's a response!" } - - before do - allow(ai_request).to receive(:request).and_return(ai_response) - allow(reader).to receive(:request).and_return(ai_response) - allow(reader).to receive(:provider_prompt_class) - .and_return(::Gitlab::Llm::Chain::Tools::JsonReader::Prompts::Anthropic) - stub_const("::Gitlab::Llm::Chain::Tools::JsonReader::Prompts::Anthropic::MAX_CHARACTERS", 4) - end - - describe '#name' do - it 'returns tool name' do - expect(described_class::NAME).to eq('ResourceReader') - end - - it 'returns tool human name' do - expect(described_class::HUMAN_NAME).to eq('Prepare data') - end - end - - describe '#description' do - it 'returns tool description' do - expect(described_class::DESCRIPTION) - .to include('Useful tool when you need to get information about specific ' \ - 'resource that was already identified. ' \ - 'Action Input for this tools always starts with: `data`') - end - end - - describe '#execute' do - before do - allow(Gitlab).to receive(:org_or_com?).and_return(true) - stub_ee_application_setting(should_check_namespace_plan: true) - allow(group.namespace_settings).to receive(:experiment_settings_allowed?).and_return(true) - stub_licensed_features( - ai_features: true, - epics: true, - experimental_features: true, - ai_chat: true - ) - group.namespace_settings.update!(experiment_features_enabled: true) - end - - context 'when the tool was already used' do - before do - context.tools_used << described_class - end - - it 'returns already used answer' do - allow(reader).to receive(:request).and_return('response') - - expect(reader.execute.content).to eq('You already have the answer from ResourceReader tool, read carefully.') - end - end - - it 'returns error when the resource is not authorized' do - expect(reader.execute.content).to eq('I am sorry, I am unable to find what you are looking for.') - end - - context 'when the resource is authorized' do - before_all do - project.add_guest(user) - end - - context 'when resource length equals or exceeds max tokens' do - it 'processes the long path' do - expect(reader).to receive(:process_long_path) - - reader.execute - end - end - - context 'when resource length does not exceed max tokens' do - before do - stub_const("::Gitlab::Llm::Chain::Tools::JsonReader::Prompts::Anthropic::MAX_CHARACTERS", 999999) - end - - it 'processes the short path' do - expect(reader).to receive(:process_short_path) - - reader.execute - end - - it "returns a final answer even if the response doesn't contain a 'final answer' token" do - issue_json = Ai::AiResource::Issue.new(issue) - .serialize_for_ai( - user: context.current_user, - content_limit: ::Gitlab::Llm::Chain::Tools::JsonReader::Prompts::Anthropic::MAX_CHARACTERS - ).to_xml(root: :root, skip_types: true, skip_instruct: true) - ai_response = "Please use this information about this resource: #{issue_json}" - - expect_answer_with_content(ai_response) - end - end - - context 'if final answer is present' do - let(:answer) { "The answer you've been looking for! Made with computers!" } - let(:ai_response) { "Final Answer: #{answer}" } - - it 'returns a ::Gitlab::Llm::Chain::Answer' do - expect_answer_with_content(answer) - end - end - - context 'when final answer is not present' do - context 'when the response contains an action' do - context 'when first action includes JsonReaderListKeys' do - let(:ai_response) do - <<~PROMPT - I should look at the keys that exist in `data` to see what I have access to - Action: JsonReaderListKeys - Action Input: data - PROMPT - end - - it 'uses `Utils::JsonReaderListKeys`' do - expect(::Gitlab::Llm::Chain::Utils::JsonReaderListKeys).to receive(:handle_keys) - - expect_and_stop_execution_at(regex: /Observation:/) - - reader.execute - end - end - - context 'when action includes JsonReaderGetValue' do - let(:ai_response) do - <<~PROMPT - I should look at the data that exist in `labels` to see what I have access to - Action: JsonReaderGetValue - Action Input: data['labels'][0] - PROMPT - end - - it 'uses `Utils::JsonReaderGetValue`' do - expect(::Gitlab::Llm::Chain::Utils::JsonReaderGetValue).to receive(:handle_keys).at_least(:once) - - expect_and_stop_execution_at(regex: /Observation:/) - - reader.execute - end - end - end - - context 'when the response contains no action' do - let(:ai_response) do - <<~PROMPT - Action Input: I'm here for the birthday party. Beep beep boop. - PROMPT - end - - it 'does not add an observation to the next recursive call' do - response = reader.execute - error_msg = "is not valid, Action must be either `JsonReaderListKeys` or `JsonReaderGetValue`" - expect(response).to be_a(Gitlab::Llm::Chain::Answer) - expect(response.content).to include(error_msg) - end - end - - context 'when the response does not contain any keywords' do - let(:ai_response) do - <<~PROMPT - I'm here for the birthday party. Beep beep boop. - PROMPT - end - - it 'returns final response' do - expect_answer_with_content(ai_response.strip) - end - end - end - - context 'when resource is not serialisable into JSON' do - let(:resource) { Class.new } - - before do - allow(reader).to receive(:authorize).and_return(true) - end - - # e.g. when `serialize_instance` hasn't been defined on a model - it 'returns an answer with an error' do - response = reader.execute - - expect(response).to be_a(Gitlab::Llm::Chain::Answer) - expect(response.content).to match(/Unexpected error/) - end - end - end - end -end - -def expect_answer_with_content(expected_content) - expected_params = { status: :ok, context: context, content: expected_content, tool: nil } - expect(::Gitlab::Llm::Chain::Answer).to receive(:new).with(expected_params).and_call_original - - expect(reader.execute).to be_a(::Gitlab::Llm::Chain::Answer) -end - -def expect_and_stop_execution_at(regex:) - # this is to make sure we get this far into the method - # but we tell the spec to stop the code before the recursive call - # which would cause a stack overflow. - allow(options[:suggestions]).to receive(:<<) - allow(options[:suggestions]).to receive(:<<).with(regex).and_raise("Stop here please") -end - -def description_content - <<~PROMPT - I think it would be good to include AI at birthday parties. It could automatically bring in the cake! - They could even do quality control on the number of candles to ensure nobody is lying about their age. - PROMPT -end - -def note_content - <<~PROMPT - AI isn't a robot though, they couldn't bring in the cake because AI is a language model and not a physical robot. - Disagree. - PROMPT -end diff --git a/ee/spec/lib/gitlab/llm/chain/tools/json_reader/prompts/anthropic_spec.rb b/ee/spec/lib/gitlab/llm/chain/tools/json_reader/prompts/anthropic_spec.rb deleted file mode 100644 index 419698ab4f46f14c0546cd624fc2849533667ea8..0000000000000000000000000000000000000000 --- a/ee/spec/lib/gitlab/llm/chain/tools/json_reader/prompts/anthropic_spec.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Llm::Chain::Tools::JsonReader::Prompts::Anthropic, feature_category: :duo_chat do - describe '.prompt' do - it 'returns prompt' do - options = { - suggestions: "some suggestions", - input: 'foo?' - } - prompt = described_class.prompt(options)[:prompt] - - expect(prompt).to include('Human:') - expect(prompt).to include('Assistant:') - expect(prompt).to include('Thought:') - expect(prompt).to include('some suggestions') - expect(prompt).to include('foo?') - expect(prompt).to include('You are an agent designed to interact with JSON.') - end - end -end diff --git a/ee/spec/lib/gitlab/llm/chain/tools/json_reader/prompts/vertex_ai_spec.rb b/ee/spec/lib/gitlab/llm/chain/tools/json_reader/prompts/vertex_ai_spec.rb deleted file mode 100644 index 9472cda97f452764e72b801fd5bb2e3469b6823f..0000000000000000000000000000000000000000 --- a/ee/spec/lib/gitlab/llm/chain/tools/json_reader/prompts/vertex_ai_spec.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Llm::Chain::Tools::JsonReader::Prompts::VertexAi, feature_category: :duo_chat do - describe '.prompt' do - it 'returns prompt' do - options = { - suggestions: "some suggestions", - input: 'foo?' - } - prompt = described_class.prompt(options)[:prompt] - - expect(prompt).to include('Thought:') - expect(prompt).to include('some suggestions') - expect(prompt).to include('foo?') - expect(prompt).to include('You are an agent designed to interact with JSON.') - end - end -end diff --git a/ee/spec/lib/gitlab/llm/completions/chat_real_requests_spec.rb b/ee/spec/lib/gitlab/llm/completions/chat_real_requests_spec.rb index 30592a7ea107f83a91fbc8dbf432c8422c6280e7..aa5339425065c1bd3e48b7ac9ea9f535a77f1dc0 100644 --- a/ee/spec/lib/gitlab/llm/completions/chat_real_requests_spec.rb +++ b/ee/spec/lib/gitlab/llm/completions/chat_real_requests_spec.rb @@ -195,10 +195,8 @@ # rubocop: disable Layout/LineLength -- keep table structure readable where(:input_template, :tools) do # evaluation of questions which involve processing of other resources is not reliable yet - # because both IssueIdentifier and JsonReader tools assume we work with single resource: + # because the IssueIdentifier tool assumes we work with single resource: # IssueIdentifier overrides context.resource - # JsonReader takes resource from context - # So JsonReader twice with different action input 'Can you provide more details about that issue?' | %w[IssueReader] 'Can you reword your answer?' | [] 'Can you simplify your answer?' | [] @@ -397,10 +395,8 @@ # rubocop: disable Layout/LineLength -- keep table structure readable where(:input_template, :tools) do # evaluation of questions which involve processing of other resources is not reliable yet - # because both EpicIdentifier and JsonReader tools assume we work with single resource: + # because the EpicIdentifier tool assumes we work with single resource: # EpicIdentifier overrides context.resource - # JsonReader takes resource from context - # So JsonReader twice with different action input 'Can you provide more details about that epic?' | %w[EpicReader] # Translation would have to be explicitly allowed in prompt rules first 'Can you reword your answer?' | []