From 58a0908879ca0d680ad320c89ba0db6e6864761f Mon Sep 17 00:00:00 2001 From: Mark Chao <mchao@gitlab.com> Date: Tue, 30 Jan 2024 07:24:44 +0000 Subject: [PATCH] Document ai_question_category event --- config/events/ai_question_category.yml | 21 +++++++++++++++++++++ doc/development/ai_features/duo_chat.md | 18 ++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 config/events/ai_question_category.yml diff --git a/config/events/ai_question_category.yml b/config/events/ai_question_category.yml new file mode 100644 index 000000000000..c08021791880 --- /dev/null +++ b/config/events/ai_question_category.yml @@ -0,0 +1,21 @@ +--- +description: "User submits a Duo Chat message. + The message is analyzed using LLM (e.g. categorization) and Ruby (e.g. message length). + Schema: https://gitlab.com/gitlab-org/iglu/-/blob/master/public/schemas/com.gitlab/ai_question_category/jsonschema + Category list: https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/gitlab/llm/fixtures/categories.xml" +category: Gitlab::Llm::Anthropic::Completions::CategorizeQuestion +action: ai_question_category +label_description: +property_description: Request ID to link to other events of the same AI request. +value_description: +identifiers: + - user +product_section: data-science +product_stage: ai-powered +product_group: 'group::duo chat' +milestone: '16.6' +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/132803 +distributions: + - ee +tiers: + - ultimate diff --git a/doc/development/ai_features/duo_chat.md b/doc/development/ai_features/duo_chat.md index 94e34a8564f3..939ce3f0656e 100644 --- a/doc/development/ai_features/duo_chat.md +++ b/doc/development/ai_features/duo_chat.md @@ -220,3 +220,21 @@ Ultimate tier, Staging Ref may be an easier place to test changes as a GitLab team member because [you can make yourself an instance Admin in Staging Ref](https://handbook.gitlab.com/handbook/engineering/infrastructure/environments/staging-ref/#admin-access) and, as an Admin, easily create licensed groups for testing. + +## Product Analysis + +To better understand how the feature is used, each production user input message is analyzed using LLM and Ruby, +and the analysis is tracked as a Snowplow event. + +The analysis can contain any of the attributes defined in the latest [iglu schema](https://gitlab.com/gitlab-org/iglu/-/blob/master/public/schemas/com.gitlab/ai_question_category/jsonschema). + +- All possible "category" and "detailed_category" are listed [here](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/gitlab/llm/fixtures/categories.xml). +- The following are yet to be implemented: + - "is_proper_sentence" + - "asked_on_the_same_page" +- The following are deprecated: + - "number_of_questions_in_history" + - "length_of_questions_in_history" + - "time_since_first_question" + +[Dashboards](https://handbook.gitlab.com/handbook/engineering/development/data-science/duo-chat/#-dashboards-internal-only) can be created to visualize the collected data. -- GitLab