diff --git a/doc/development/ai_features/index.md b/doc/development/ai_features/index.md
index 272975b3cb916935136086298e07d4de9151f1f4..8ad87a0d8c06c54c1e57d51bb7395ffddb4247bb 100644
--- a/doc/development/ai_features/index.md
+++ b/doc/development/ai_features/index.md
@@ -164,6 +164,15 @@ If you currently run you local GDK as Self-Managed (default for GDK), no argumen
It's recommended to run `gdk restart` after the task succeeded.
+If you need to use evaluation framework (as described [here](https://gitlab.com/gitlab-org/modelops/ai-model-validation-and-research/ai-evaluation/prompt-library/-/blob/main/doc/how-to/run_duo_chat_eval.md?ref_type=heads#evaluation-on-issueepic))
+you can run special Rake task: `GITLAB_SIMULATE_SAAS=1 bundle exec 'rake gitlab:duo:setup_evaluation[<test-group-name>]'`.
+It repeats steps from original setup Rake task, and also imports specially prepared groups and projects.
+Since we use `Setup` class (under `ee/lib/gitlab/duo/developments/setup.rb`) that requires "saas" mode to create a group
+(necessary for importing subgroups), you need to set `GITLAB_SIMULATE_SAAS=1` if it's currently `GITLAB_SIMULATE_SAAS=0`.
+This is just to complete the import successfully, and then you can switch back to `GITLAB_SIMULATE_SAAS=0`.
+To run this task, your GDK server must be running. After running this Rake task, import process will be in progress for
+said groups and projects.
+
### Recommended: Set `CLOUD_CONNECTOR_SELF_SIGN_TOKENS` environment variable
If you plan to run you local GDK as Self-Managed (for GDK), it is recommended to set this environment variable.
diff --git a/ee/lib/gitlab/duo/developments/setup.rb b/ee/lib/gitlab/duo/developments/setup.rb
index 09781575e6571071b25f24e44409ffe974c9582c..ac34445c3323319bd8fb806dc9b3ccb254481a47 100644
--- a/ee/lib/gitlab/duo/developments/setup.rb
+++ b/ee/lib/gitlab/duo/developments/setup.rb
@@ -205,6 +205,8 @@ def print_result
For more development guidelines, see https://docs.gitlab.com/ee/development/ai_features/index.html.
MSG
+
+ Group.find_by_full_path(@namespace)
end
end
end
diff --git a/ee/lib/gitlab/duo/developments/setup_groups_for_model_evaluation.rb b/ee/lib/gitlab/duo/developments/setup_groups_for_model_evaluation.rb
new file mode 100644
index 0000000000000000000000000000000000000000..a21dc886c8f82964978432148b4efaf8f11b2244
--- /dev/null
+++ b/ee/lib/gitlab/duo/developments/setup_groups_for_model_evaluation.rb
@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Duo
+ module Developments
+ class SetupGroupsForModelEvaluation
+ STRUCTURE = {
+ 'gitlab_com' => { projects: ['www-gitlab-com'], name: 'gitlab-com' },
+ 'gitlab_org' => { projects: ['gitlab'], name: 'gitlab-org' }
+ }.freeze
+ DOWNLOAD_FOLDER = 'tmp'
+ SAMPLES_FOLDER = 'duo_chat_samples'
+ GROUP_FILE_NAME = '01_group.tar.gz'
+ FILE_NAME = 'duo_chat_samples.tar.gz'
+ DOWNLOAD_URL = 'https://gitlab.com/gitlab-org/ai-powered/datasets/-/package_files/135727282/download'
+ GROUP_IMPORT_URL = '/api/v4/groups/import'
+ PROJECT_IMPORT_URL = '/api/v4/projects/import'
+
+ def initialize(group)
+ @main_group = group
+ @current_user = User.find_by(username: 'root') # rubocop:disable CodeReuse/ActiveRecord -- we need admin user
+ end
+
+ def execute
+ ensure_dev_mode!
+ set_token!
+ ensure_server_running!
+ download_and_unpack_file
+ create_subgroups
+ create_subprojects
+ delete_temporary_directory!
+ clean_up_token!
+
+ print_output
+ end
+
+ private
+
+ attr_reader :main_group, :current_user, :token_value, :token
+
+ # rubocop:disable Style/GuardClause -- Keep it explicit
+ def ensure_dev_mode!
+ unless ::Gitlab.dev_or_test_env?
+ raise <<~MSG
+ Setup can only be performed in development or test environment, however, the current environment is #{ENV['RAILS_ENV']}.
+ MSG
+ end
+ end
+ # rubocop:enable Style/GuardClause
+
+ def set_token!
+ @token = current_user.personal_access_tokens.create(scopes: ['api'], name: 'Automation token',
+ expires_at: 1.day.from_now)
+ @token_value = "token-string-#{SecureRandom.hex(10)}"
+ @token.set_token(token_value)
+ @token.save!
+ end
+
+ def clean_up_token!
+ token.destroy!
+ end
+
+ def ensure_server_running!
+ return true if Gitlab::HTTP.get(instance_url).success?
+
+ raise 'Server is not running, please start your GitLab server'
+ end
+
+ def download_and_unpack_file
+ download_path = Rails.root.join(DOWNLOAD_FOLDER, FILE_NAME)
+
+ download_file(DOWNLOAD_URL, download_path)
+ unzip_file(DOWNLOAD_FOLDER, FILE_NAME)
+
+ FileUtils.rm(download_path)
+ end
+
+ def download_file(url, path)
+ File.open(path, 'wb') do |file|
+ file.write(Gitlab::HTTP.get(url).parsed_response)
+ end
+ end
+
+ def unzip_file(download_folder, file_name)
+ Dir.chdir(Rails.root.join(download_folder)) do
+ `tar -xzvf #{file_name}`
+ end
+ end
+
+ def create_subprojects
+ STRUCTURE.each do |name, structure|
+ structure[:projects].each do |project|
+ project_file_name = "02_#{project.tr('-', '_')}.tar.gz"
+ file = Rails.root.join(DOWNLOAD_FOLDER, SAMPLES_FOLDER, name, project_file_name)
+ namespace = main_group.children.find_by(name: structure[:name]) # rubocop:disable CodeReuse/ActiveRecord -- we need to find a group by name
+ create_subproject(name: project, file: file, namespace_id: namespace.id)
+ end
+ end
+ end
+
+ def create_subgroups
+ STRUCTURE.each do |name, structure|
+ file = Rails.root.join(DOWNLOAD_FOLDER, SAMPLES_FOLDER, name, GROUP_FILE_NAME)
+ create_subgroup(name: structure[:name], file: file)
+ end
+ end
+
+ def create_subgroup(params)
+ url = "#{instance_url}#{GROUP_IMPORT_URL}"
+
+ headers = {
+ 'PRIVATE-TOKEN' => token_value
+ }
+ body = {
+ name: params[:name],
+ path: params[:name],
+ file: File.new(params[:file]),
+ parent_id: main_group.id
+ }
+
+ response = Gitlab::HTTP.post(url, headers: headers, body: body)
+
+ puts "API response for #{params[:name]} import"
+ puts response.body
+ end
+
+ def create_subproject(params)
+ url = "#{instance_url}#{PROJECT_IMPORT_URL}"
+
+ headers = {
+ 'PRIVATE-TOKEN' => token_value
+ }
+ body = {
+ name: params[:name],
+ path: params[:name],
+ file: File.new(params[:file]),
+ namespace: params[:namespace_id]
+ }
+
+ response = Gitlab::HTTP.post(url, headers: headers, body: body)
+
+ puts "API response for #{params[:name]} import"
+ puts response.body
+ end
+
+ def instance_url
+ "#{Gitlab.config.gitlab.protocol}://#{Gitlab.config.gitlab.host}:#{Gitlab.config.gitlab.port}"
+ end
+
+ def delete_temporary_directory!
+ FileUtils.rm_rf(Rails.root.join(DOWNLOAD_FOLDER, SAMPLES_FOLDER))
+ end
+
+ def print_output
+ puts <<~MSG
+ ----------------------------------------
+ Setup for evaluation Complete!
+ ----------------------------------------
+
+ Visit "#{Gitlab.config.gitlab.protocol}://#{Gitlab.config.gitlab.host}:#{Gitlab.config.gitlab.port}/#{main_group.full_path}"
+ and please see if the subgroups structure looks like:
+ |
+ - gitlab-com
+ | - www-gitlab-com
+ |
+ - gitlab-org
+ | - gitlab
+ MSG
+ end
+ end
+ end
+ end
+end
diff --git a/ee/lib/tasks/gitlab/duo.rake b/ee/lib/tasks/gitlab/duo.rake
index 41ebcf8082bf7c9872ff7ad413f1d107ada19b50..26286170a65a95dddc332abb4bcb397d2d1d26a1 100644
--- a/ee/lib/tasks/gitlab/duo.rake
+++ b/ee/lib/tasks/gitlab/duo.rake
@@ -11,5 +11,11 @@ namespace :gitlab do
task enable_feature_flags: :gitlab_environment do
Gitlab::Duo::Developments::FeatureFlagEnabler.execute
end
+
+ desc 'GitLab | Duo | Create evaluation-ready group'
+ task :setup_evaluation, [:root_group_path] => :environment do |_, args|
+ group = Gitlab::Duo::Developments::Setup.new(args).execute
+ Gitlab::Duo::Developments::SetupGroupsForModelEvaluation.new(group).execute
+ end
end
end
diff --git a/ee/spec/lib/gitlab/duo/developments/setup_groups_for_model_evaluation_spec.rb b/ee/spec/lib/gitlab/duo/developments/setup_groups_for_model_evaluation_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..712ec2b1ee67255aa67f49074161822156822129
--- /dev/null
+++ b/ee/spec/lib/gitlab/duo/developments/setup_groups_for_model_evaluation_spec.rb
@@ -0,0 +1,174 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Duo::Developments::SetupGroupsForModelEvaluation, :saas, :gitlab_duo, :silence_stdout, feature_category: :duo_chat do
+ let_it_be(:user) { create(:user, username: 'root') }
+ let(:group) { create(:group) }
+ let(:setup_evaluation) { described_class.new(group) }
+ let(:http_response) { instance_double(HTTParty::Response, body: 'File content') }
+ let(:file_double) { instance_double(File) }
+
+ before do
+ allow(SecureRandom).to receive(:hex).and_return('1')
+ end
+
+ describe '#execute' do
+ context 'when the server is running' do
+ before do
+ allow(http_response).to receive(:success?).and_return(true)
+ allow(http_response).to receive(:parsed_response).and_return({})
+ allow(Gitlab::HTTP).to receive(:get).and_return(http_response)
+ allow(Gitlab::HTTP).to receive(:get).with("https://gitlab.com/gitlab-org/ai-powered/datasets/-/package_files/135727282/download")
+ .and_return(http_response)
+ end
+
+ it 'goes through the process' do
+ expect(setup_evaluation).to receive(:set_token!)
+ expect(setup_evaluation).to receive(:ensure_server_running!)
+ expect(setup_evaluation).to receive(:download_and_unpack_file)
+ expect(setup_evaluation).to receive(:create_subgroups)
+ expect(setup_evaluation).to receive(:create_subprojects)
+ expect(setup_evaluation).to receive(:delete_temporary_directory!)
+ expect(setup_evaluation).to receive(:clean_up_token!)
+ expect(setup_evaluation).to receive(:print_output)
+
+ setup_evaluation.execute
+ end
+
+ describe '#set_token!' do
+ it 'creates token' do
+ expect { setup_evaluation.send(:set_token!) }.to change { PersonalAccessToken.count }.by(1)
+ end
+ end
+
+ describe '#clean_up_token!' do
+ it 'deletes token' do
+ setup_evaluation.send(:set_token!)
+
+ expect { setup_evaluation.send(:clean_up_token!) }.to change { PersonalAccessToken.count }.by(-1)
+ end
+ end
+
+ describe '#download_and_unpack_file' do
+ it 'unzips the file' do
+ expect(setup_evaluation).to receive(:unzip_file).with('tmp', 'duo_chat_samples.tar.gz')
+
+ setup_evaluation.send(:download_and_unpack_file)
+ end
+
+ it 'runs through files' do
+ expect(FileUtils).to receive(:rm)
+
+ setup_evaluation.send(:download_and_unpack_file)
+ end
+ end
+
+ describe '#delete_temporary_directory!' do
+ it 'deletes folder' do
+ expect(FileUtils).to receive(:rm_rf).at_least(:once)
+
+ setup_evaluation.send(:delete_temporary_directory!)
+ end
+ end
+
+ describe '#create_subgroups' do
+ it 'creates subgroups' do
+ expect(setup_evaluation).to receive(:create_subgroup).with(name: 'gitlab-com',
+ file: Rails.root.join("tmp/duo_chat_samples/gitlab_com/01_group.tar.gz"))
+ expect(setup_evaluation).to receive(:create_subgroup).with(name: 'gitlab-org',
+ file: Rails.root.join("tmp/duo_chat_samples/gitlab_org/01_group.tar.gz"))
+
+ setup_evaluation.send(:create_subgroups)
+ end
+ end
+
+ describe '#create_subprojects' do
+ it 'creates subprojects' do
+ gitlab_com_group = create(:group, name: 'gitlab-com', parent: group)
+ gitlab_org_group = create(:group, name: 'gitlab-org', parent: group)
+
+ expect(setup_evaluation).to receive(:create_subproject).with(
+ name: 'www-gitlab-com',
+ file: Rails.root.join("tmp/duo_chat_samples/gitlab_com/02_www_gitlab_com.tar.gz"),
+ namespace_id: gitlab_com_group.id
+ )
+ expect(setup_evaluation).to receive(:create_subproject).with(
+ name: 'gitlab',
+ file: Rails.root.join("tmp/duo_chat_samples/gitlab_org/02_gitlab.tar.gz"),
+ namespace_id: gitlab_org_group.id
+ )
+
+ setup_evaluation.send(:create_subprojects)
+ end
+ end
+
+ describe '#create_subgroup' do
+ it 'creates a subgroup' do
+ file = Rails.root.join("tmp/duo_chat_samples/gitlab_com/01_group.tar.gz")
+ body = { name: 'gitlab-com', path: 'gitlab-com', parent_id: group.id, file: file_double }
+
+ expect(File).to receive(:new).with(file).and_return(file_double)
+ expect(setup_evaluation).to receive(:token_value).and_return('token-string-1')
+
+ expect(Gitlab::HTTP).to receive(:post)
+ .with("#{setup_evaluation.send(:instance_url)}/api/v4/groups/import",
+ headers: { 'PRIVATE-TOKEN' => 'token-string-1' }, body: hash_including(**body))
+ .and_return(http_response)
+
+ setup_evaluation.send(:create_subgroup, name: 'gitlab-com', file: file)
+ end
+ end
+
+ describe '#create_subproject' do
+ it 'creates a subproject' do
+ gitlab_com_group = create(:group, name: 'gitlab-com', parent: group)
+
+ file = Rails.root.join("tmp/duo_chat_samples/gitlab_com/02_www_gitlab_com.tar.gz")
+ body = { name: 'www-gitlab-com', path: 'www-gitlab-com', namespace: gitlab_com_group.id, file: file_double }
+
+ expect(File).to receive(:new).with(file).and_return(file_double)
+ expect(setup_evaluation).to receive(:token_value).and_return('token-string-1')
+
+ expect(Gitlab::HTTP).to receive(:post)
+ .with("#{setup_evaluation.send(:instance_url)}/api/v4/projects/import",
+ body: hash_including(**body), headers: { 'PRIVATE-TOKEN' => 'token-string-1' })
+ .and_return(http_response)
+
+ setup_evaluation.send(:create_subproject, name: 'www-gitlab-com', file: file,
+ namespace_id: gitlab_com_group.id)
+ end
+ end
+
+ context 'when running not in dev or test mode' do
+ before do
+ stub_env('RAILS_ENV', 'production')
+ allow(Gitlab).to receive(:dev_or_test_env?).and_return(false)
+ end
+
+ it 'raises an error' do
+ expect { setup_evaluation.execute }.to raise_error(RuntimeError)
+ end
+ end
+
+ context 'when finished working' do
+ it 'shows a message' do
+ expect do
+ setup_evaluation.send(:print_output)
+ end.to output(a_string_including('Setup for evaluation Complete!')).to_stdout
+ end
+ end
+ end
+
+ context 'when the server is not running' do
+ before do
+ allow(http_response).to receive(:success?).and_return(false)
+ allow(Gitlab::HTTP).to receive(:get).and_return(http_response)
+ end
+
+ it 'raises an error' do
+ expect { setup_evaluation.execute }.to raise_error('Server is not running, please start your GitLab server')
+ end
+ end
+ end
+end