Add job to trigger code-suggestion-scenarios

- Adds a job that triggers a multi-project pipeline in the
code-suggestion-scenarios project

.gitlab/ci/qa.gitlab-ci.yml

@@ -211,3 +211,42 @@ e2e:test-on-gdk:
     DYNAMIC_PIPELINE_YML: test-on-gdk-pipeline.yml
     SKIP_MESSAGE: Skipping test-on-gdk due to mr containing only quarantine changes!
     GDK_IMAGE: "${CI_REGISTRY_IMAGE}/gitlab-qa-gdk:${CI_COMMIT_SHA}"
+
+e2e:code-suggestions-eval:
+  extends:
+    - .qa:rules:code-suggestions-eval
+  stage: qa
+  needs: ["build-gdk-image"]
+  variables:
+    CS_EVAL_DOWNSTREAM_BRANCH: main
+    GITLAB_SHA: $CI_COMMIT_SHA
+  trigger:
+    strategy: depend
+    forward:
+      yaml_variables: true
+      pipeline_variables: true
+    project: gitlab-com/create-stage/code-creation/code-suggestion-scenarios
+    branch: $CS_EVAL_DOWNSTREAM_BRANCH
+
+e2e:code-suggestions-eval-results:
+  extends:
+    - .default-retry
+    - .qa:rules:code-suggestions-eval-results
+  stage: post-qa
+  needs:
+    - e2e:code-suggestions-eval
+  variables:
+    TRIGGER_JOB_NAME: "e2e:code-suggestions-eval"
+    DOWNSTREAM_PROJECT: gitlab-com/create-stage/code-creation/code-suggestion-scenarios
+    DOWNSTREAM_JOB_NAME: run_scenarios
+    DOWNSTREAM_JOB_ARTIFACT_PATH: scores-DOWNSTREAM_JOB_ID.csv
+    OUTPUT_ARTIFACT_PATH: scores.csv
+  before_script:
+    - source scripts/utils.sh
+    - install_gitlab_gem
+  script:
+    - scripts/download-downstream-artifact.rb
+  artifacts:
+    expose_as: 'Code Suggestions evaluation results'
+    paths:
+      - scores.csv

.gitlab/ci/rules.gitlab-ci.yml

@@ -92,6 +92,9 @@
 .if-merge-request-labels-run-review-app: &if-merge-request-labels-run-review-app
   if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:run-review-app/'
 
+.if-merge-request-labels-run-cs-evaluation: &if-merge-request-labels-run-cs-evaluation
+  if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:run-CS-evaluation/'
+
 .if-merge-request-labels-skip-undercoverage: &if-merge-request-labels-skip-undercoverage
   if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:skip-undercoverage/'
 
@@ -950,6 +953,7 @@
     - <<: *if-merge-request
       changes: *dependency-patterns
     - <<: *if-merge-request-labels-run-all-e2e
+    - <<: *if-merge-request-labels-run-cs-evaluation
     - <<: *if-merge-request
       changes: *feature-flag-development-config-patterns
     - <<: *if-merge-request
@@ -1675,6 +1679,27 @@
   rules:
     - <<: [*if-dot-com-gitlab-org-schedule, *qa-e2e-test-schedule-variables]
 
+.qa:rules:code-suggestions-eval-base:
+  rules:
+    - !reference [".strict-ee-only-rules", rules]
+    - <<: *if-fork-merge-request
+      when: never
+    - <<: *if-merge-request-labels-run-cs-evaluation
+
+.qa:rules:code-suggestions-eval:
+  rules:
+    - !reference [".qa:rules:code-suggestions-eval-base", rules]
+    - <<: *if-merge-request
+      changes: *code-patterns
+      when: manual
+      allow_failure: true
+
+.qa:rules:code-suggestions-eval-results:
+  rules:
+    - !reference [".qa:rules:code-suggestions-eval-base", rules]
+    - <<: *if-merge-request
+      changes: *code-patterns
+
 # Note: If any changes are made to this rule, the following should also be updated:
 # 1) .qa:rules:manual-omnibus-and-follow-up-e2e
 # 2) .qa:rules:follow-up-e2e

doc/development/pipelines/internals.md

@@ -219,6 +219,7 @@ and included in `rules` definitions via [YAML anchors](../../ci/yaml/yaml_optimi
 | `if-merge-request-title-as-if-foss`                          | Matches if the pipeline is for a merge request and the MR has label ~"pipeline:run-as-if-foss" | |
 | `if-merge-request-title-update-caches`                       | Matches if the pipeline is for a merge request and the MR has label ~"pipeline:update-cache". | |
 | `if-merge-request-labels-run-all-rspec`                      | Matches if the pipeline is for a merge request and the MR has label ~"pipeline:run-all-rspec". | |
+| `if-merge-request-labels-run-cs-evaluation`                  | Matches if the pipeline is for a merge request and the MR has label ~"pipeline:run-CS-evaluation". | |
 | `if-security-merge-request`                                  | Matches if the pipeline is for a security merge request. | |
 | `if-security-schedule`                                       | Matches if the pipeline is for a security scheduled pipeline. | |
 | `if-nightly-master-schedule`                                 | Matches if the pipeline is for a `master` scheduled pipeline with `$NIGHTLY` set. | |

scripts/download-downstream-artifact.rb

+#!/usr/bin/env ruby
+
+# frozen_string_literal: true
+
+require 'gitlab'
+
+require_relative 'api/default_options'
+
+# This class allows an upstream job to fetch an artifact from a job in a downstream pipeline.
+#
+# Until https://gitlab.com/gitlab-org/gitlab/-/issues/285100 is resolved it's not straightforward for an upstream
+# pipeline to use artifacts from a downstream pipeline. There is a workaround for parent-child pipelines (see the issue)
+# but it relies on CI_MERGE_REQUEST_REF_PATH so it doesn't work for multi-project pipelines.
+#
+# This uses the Jobs API to get pipeline bridges (trigger jobs) and the Job artifacts API to download artifacts.
+# - https://docs.gitlab.com/ee/api/jobs.html#list-pipeline-trigger-jobs
+# - https://docs.gitlab.com/ee/api/job_artifacts.html
+#
+# Note: This class also works for parent-child pipelines within the same project, it's just not necessary in that case.
+class DownloadDownstreamArtifact
+  def initialize(options)
+    @upstream_project = options.fetch(:upstream_project, API::DEFAULT_OPTIONS[:project])
+    @upstream_pipeline_id = options.fetch(:upstream_pipeline_id, API::DEFAULT_OPTIONS[:pipeline_id])
+    @downstream_project = options.fetch(:downstream_project, API::DEFAULT_OPTIONS[:project])
+    @downstream_job_name = options.fetch(:downstream_job_name)
+    @trigger_job_name = options.fetch(:trigger_job_name)
+    @downstream_artifact_path = options.fetch(:downstream_artifact_path)
+    @output_artifact_path = options.fetch(:output_artifact_path)
+
+    unless options.key?(:api_token)
+      raise ArgumentError, 'PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE is required to access downstream pipelines'
+    end
+
+    api_token = options.fetch(:api_token)
+
+    @client = Gitlab.client(
+      endpoint: options.fetch(:endpoint, API::DEFAULT_OPTIONS[:endpoint]),
+      private_token: api_token
+    )
+  end
+
+  def execute
+    unless downstream_pipeline
+      abort("Could not find downstream pipeline triggered via #{trigger_job_name} in project #{downstream_project}")
+    end
+
+    unless downstream_job
+      abort("Could not find job with name '#{downstream_job_name}' in #{downstream_pipeline['web_url']}")
+    end
+
+    puts "Fetching scores artifact from downstream pipeline triggered via #{trigger_job_name}..."
+    puts "Downstream pipeline is #{downstream_pipeline['web_url']}."
+    puts %(Downstream job "#{downstream_job_name}": #{downstream_job['web_url']}.)
+
+    path = downstream_artifact_path.sub('DOWNSTREAM_JOB_ID', downstream_job.id.to_s)
+    puts %(Fetching artifact "#{path}" from #{downstream_job_name}...)
+
+    download_and_save_artifact(path)
+
+    puts "Artifact saved as #{output_artifact_path} ..."
+  end
+
+  def self.options_from_env
+    API::DEFAULT_OPTIONS.merge({
+      upstream_project: API::DEFAULT_OPTIONS[:project],
+      upstream_pipeline_id: API::DEFAULT_OPTIONS[:pipeline_id],
+      downstream_project: ENV.fetch('DOWNSTREAM_PROJECT', API::DEFAULT_OPTIONS[:project]),
+      downstream_job_name: ENV['DOWNSTREAM_JOB_NAME'],
+      trigger_job_name: ENV['TRIGGER_JOB_NAME'],
+      downstream_artifact_path: ENV['DOWNSTREAM_JOB_ARTIFACT_PATH'],
+      output_artifact_path: ENV['OUTPUT_ARTIFACT_PATH']
+    }).except(:project, :pipeline_id)
+  end
+
+  private
+
+  attr_reader :downstream_artifact_path,
+    :output_artifact_path,
+    :downstream_job_name,
+    :trigger_job_name,
+    :upstream_project,
+    :downstream_project,
+    :upstream_pipeline_id,
+    :client
+
+  def bridge
+    @bridge ||= client
+      .pipeline_bridges(upstream_project, upstream_pipeline_id, per_page: 100)
+      .auto_paginate
+      .find { |job| job.name.include?(trigger_job_name) }
+  end
+
+  def downstream_pipeline
+    @downstream_pipeline ||=
+      if bridge&.downstream_pipeline.nil?
+        nil
+      else
+        client.pipeline(downstream_project, bridge.downstream_pipeline.id)
+      end
+  end
+
+  def downstream_job
+    @downstream_job ||= client
+      .pipeline_jobs(downstream_project, downstream_pipeline.id)
+      .find { |job| job.name.include?(downstream_job_name) }
+  end
+
+  def download_and_save_artifact(job_artifact_path)
+    file_response = client.download_job_artifact_file(downstream_project, downstream_job.id, job_artifact_path)
+
+    file_response.respond_to?(:read) || abort("Could not download artifact. Request returned: #{file_response}")
+
+    File.write(output_artifact_path, file_response.read)
+  end
+end
+
+if $PROGRAM_NAME == __FILE__
+  options = DownloadDownstreamArtifact.options_from_env
+
+  DownloadDownstreamArtifact.new(options).execute
+end

spec/scripts/download_downstream_artifact_spec.rb

+# frozen_string_literal: true
+
+require 'fast_spec_helper'
+require 'gitlab/rspec/all'
+require_relative '../../scripts/download-downstream-artifact'
+
+# rubocop:disable RSpec/VerifiedDoubles -- doubles are simple mocks of a few methods from external code
+
+RSpec.describe DownloadDownstreamArtifact, feature_category: :tooling do
+  include StubENV
+
+  subject(:execute) { described_class.new(options).execute }
+
+  before do
+    stub_env('PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE', nil)
+    stub_env('CI_PROJECT_ID', nil)
+    stub_env('CI_PIPELINE_ID', nil)
+    stub_env('CI_API_V4_URL', nil)
+    stub_env('DOWNSTREAM_PROJECT', nil)
+    stub_env('DOWNSTREAM_JOB_NAME', nil)
+    stub_env('TRIGGER_JOB_NAME', nil)
+    stub_env('DOWNSTREAM_JOB_ARTIFACT_PATH', nil)
+    stub_env('OUTPUT_ARTIFACT_PATH', nil)
+  end
+
+  describe '#execute' do
+    let(:options) do
+      {
+        api_token: 'asdf1234',
+        endpoint: 'https://gitlab.com/api/v4',
+        upstream_project: 'upstream/project',
+        upstream_pipeline_id: 123,
+        downstream_project: 'downstream/project',
+        downstream_job_name: 'test-job',
+        trigger_job_name: 'trigger-job',
+        downstream_artifact_path: 'scores-DOWNSTREAM_JOB_ID.csv',
+        output_artifact_path: 'scores.csv'
+      }
+    end
+
+    let(:client) { double('Gitlab::Client') }
+    let(:artifact_response) { double('io', read: 'artifact content') }
+
+    let(:job) do
+      Struct.new(:id, :name, :web_url).new(789, 'test-job', 'https://example.com/jobs/789')
+    end
+
+    let(:downstream_pipeline) do
+      Struct.new(:id, :web_url).new(111, 'https://example.com/pipelines/111')
+    end
+
+    let(:pipeline_bridges) do
+      double('pipeline_bridges', auto_paginate: [double(name: 'trigger-job', downstream_pipeline: downstream_pipeline)])
+    end
+
+    let(:expected_output) do
+      <<~OUTPUT
+        Fetching scores artifact from downstream pipeline triggered via trigger-job...
+        Downstream pipeline is https://example.com/pipelines/111.
+        Downstream job "test-job": https://example.com/jobs/789.
+        Fetching artifact "scores-789.csv" from test-job...
+        Artifact saved as scores.csv ...
+      OUTPUT
+    end
+
+    before do
+      allow(Gitlab).to receive(:client)
+        .with(endpoint: options[:endpoint], private_token: options[:api_token])
+        .and_return(client)
+
+      allow(client).to receive(:pipeline_bridges).and_return(pipeline_bridges)
+      allow(client).to receive(:pipeline).and_return(downstream_pipeline)
+      allow(client).to receive(:pipeline_jobs).and_return([job])
+      allow(client).to receive(:download_job_artifact_file).and_return(artifact_response)
+      allow(File).to receive(:write)
+    end
+
+    it 'downloads artifact from downstream pipeline' do
+      expect(client).to receive(:download_job_artifact_file).with('downstream/project', 789, 'scores-789.csv')
+
+      expect { execute }.to output(expected_output).to_stdout
+    end
+
+    it 'saves artifact to output path' do
+      expect(File).to receive(:write).with('scores.csv', 'artifact content')
+
+      expect { execute }.to output(expected_output).to_stdout
+    end
+
+    context 'when options come from environment variables' do
+      before do
+        stub_env('PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE', 'asdf1234')
+        stub_env('CI_PROJECT_ID', 'upstream/project')
+        stub_env('CI_PIPELINE_ID', '123')
+        stub_env('CI_API_V4_URL', 'https://gitlab.com/api/v4')
+        stub_env('DOWNSTREAM_PROJECT', 'downstream/project')
+        stub_env('DOWNSTREAM_JOB_NAME', 'test-job')
+        stub_env('TRIGGER_JOB_NAME', 'trigger-job')
+        stub_env('DOWNSTREAM_JOB_ARTIFACT_PATH', 'scores-DOWNSTREAM_JOB_ID.csv')
+        stub_env('OUTPUT_ARTIFACT_PATH', 'scores.csv')
+
+        stub_const('API::DEFAULT_OPTIONS', {
+          project: ENV['CI_PROJECT_ID'],
+          pipeline_id: ENV['CI_PIPELINE_ID'],
+          api_token: ENV['PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE'],
+          endpoint: ENV['CI_API_V4_URL']
+        })
+      end
+
+      it 'uses the environment variable values' do
+        options = described_class.options_from_env
+
+        expect(File).to receive(:write)
+        expect { described_class.new(options).execute }.to output(expected_output).to_stdout
+      end
+    end
+
+    context 'when the downstream pipeline cannot be found' do
+      let(:pipeline_bridges) do
+        double('pipeline_bridges', auto_paginate: [double(name: 'trigger-job', downstream_pipeline: nil)])
+      end
+
+      it 'aborts' do
+        expect(File).not_to receive(:write)
+        expect { described_class.new(options).execute }
+          .to output(
+            %r{Could not find downstream pipeline triggered via trigger-job in project downstream/project}
+          ).to_stderr
+          .and raise_error(SystemExit)
+      end
+    end
+
+    context 'when the downstream job cannot be found' do
+      let(:job) { double('job', name: 'foo') }
+
+      it 'aborts' do
+        expect(File).not_to receive(:write)
+        expect { described_class.new(options).execute }
+          .to output(
+            %r{Could not find job with name 'test-job' in https://example.com/pipelines/111}
+          ).to_stderr
+          .and raise_error(SystemExit)
+      end
+    end
+
+    context 'when the downstream artifact cannot be found' do
+      let(:artifact_response) { 'error' }
+
+      it 'aborts' do
+        expect(File).not_to receive(:write)
+        expect { described_class.new(options).execute }
+          .to output(
+            %r{Could not download artifact. Request returned: error}
+          ).to_stderr
+          .and raise_error(SystemExit)
+      end
+    end
+  end
+
+  context 'when called without an API token' do
+    let(:options) do
+      {
+        endpoint: 'https://gitlab.com/api/v4',
+        upstream_project: 'upstream/project',
+        upstream_pipeline_id: 123,
+        downstream_project: 'downstream/project',
+        downstream_job_name: 'test-job',
+        trigger_job_name: 'trigger-job',
+        downstream_artifact_path: 'scores-DOWNSTREAM_JOB_ID.csv',
+        output_artifact_path: 'scores.csv'
+      }
+    end
+
+    it 'raises an error' do
+      expect { described_class.new(options) }.to raise_error(ArgumentError)
+    end
+  end
+end
+
+# rubocop:enable RSpec/VerifiedDoubles