diff --git a/.gitlab/ci/qa.gitlab-ci.yml b/.gitlab/ci/qa.gitlab-ci.yml index a77a8bfd25282637f747c7393ad4702d17f1751a..d6249e42e811d0f105099047d48407f120a4a765 100644 --- a/.gitlab/ci/qa.gitlab-ci.yml +++ b/.gitlab/ci/qa.gitlab-ci.yml @@ -211,3 +211,42 @@ e2e:test-on-gdk: DYNAMIC_PIPELINE_YML: test-on-gdk-pipeline.yml SKIP_MESSAGE: Skipping test-on-gdk due to mr containing only quarantine changes! GDK_IMAGE: "${CI_REGISTRY_IMAGE}/gitlab-qa-gdk:${CI_COMMIT_SHA}" + +e2e:code-suggestions-eval: + extends: + - .qa:rules:code-suggestions-eval + stage: qa + needs: ["build-gdk-image"] + variables: + CS_EVAL_DOWNSTREAM_BRANCH: main + GITLAB_SHA: $CI_COMMIT_SHA + trigger: + strategy: depend + forward: + yaml_variables: true + pipeline_variables: true + project: gitlab-com/create-stage/code-creation/code-suggestion-scenarios + branch: $CS_EVAL_DOWNSTREAM_BRANCH + +e2e:code-suggestions-eval-results: + extends: + - .default-retry + - .qa:rules:code-suggestions-eval-results + stage: post-qa + needs: + - e2e:code-suggestions-eval + variables: + TRIGGER_JOB_NAME: "e2e:code-suggestions-eval" + DOWNSTREAM_PROJECT: gitlab-com/create-stage/code-creation/code-suggestion-scenarios + DOWNSTREAM_JOB_NAME: run_scenarios + DOWNSTREAM_JOB_ARTIFACT_PATH: scores-DOWNSTREAM_JOB_ID.csv + OUTPUT_ARTIFACT_PATH: scores.csv + before_script: + - source scripts/utils.sh + - install_gitlab_gem + script: + - scripts/download-downstream-artifact.rb + artifacts: + expose_as: 'Code Suggestions evaluation results' + paths: + - scores.csv diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index bca285d3f143460b33cfd28ce6745b1947b652b7..cbb65d52e6be50e6b5a6fcf54214b5b6f5978ed1 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -92,6 +92,9 @@ .if-merge-request-labels-run-review-app: &if-merge-request-labels-run-review-app if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:run-review-app/' +.if-merge-request-labels-run-cs-evaluation: &if-merge-request-labels-run-cs-evaluation + if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:run-CS-evaluation/' + .if-merge-request-labels-skip-undercoverage: &if-merge-request-labels-skip-undercoverage if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:skip-undercoverage/' @@ -950,6 +953,7 @@ - <<: *if-merge-request changes: *dependency-patterns - <<: *if-merge-request-labels-run-all-e2e + - <<: *if-merge-request-labels-run-cs-evaluation - <<: *if-merge-request changes: *feature-flag-development-config-patterns - <<: *if-merge-request @@ -1675,6 +1679,27 @@ rules: - <<: [*if-dot-com-gitlab-org-schedule, *qa-e2e-test-schedule-variables] +.qa:rules:code-suggestions-eval-base: + rules: + - !reference [".strict-ee-only-rules", rules] + - <<: *if-fork-merge-request + when: never + - <<: *if-merge-request-labels-run-cs-evaluation + +.qa:rules:code-suggestions-eval: + rules: + - !reference [".qa:rules:code-suggestions-eval-base", rules] + - <<: *if-merge-request + changes: *code-patterns + when: manual + allow_failure: true + +.qa:rules:code-suggestions-eval-results: + rules: + - !reference [".qa:rules:code-suggestions-eval-base", rules] + - <<: *if-merge-request + changes: *code-patterns + # Note: If any changes are made to this rule, the following should also be updated: # 1) .qa:rules:manual-omnibus-and-follow-up-e2e # 2) .qa:rules:follow-up-e2e diff --git a/doc/development/pipelines/internals.md b/doc/development/pipelines/internals.md index 04c1d1f32e89b0d52df3722985b0e8897fe0d5f1..df9a9d9c4ad1f4fa96793b959da42e59ce2b455a 100644 --- a/doc/development/pipelines/internals.md +++ b/doc/development/pipelines/internals.md @@ -219,6 +219,7 @@ and included in `rules` definitions via [YAML anchors](../../ci/yaml/yaml_optimi | `if-merge-request-title-as-if-foss` | Matches if the pipeline is for a merge request and the MR has label ~"pipeline:run-as-if-foss" | | | `if-merge-request-title-update-caches` | Matches if the pipeline is for a merge request and the MR has label ~"pipeline:update-cache". | | | `if-merge-request-labels-run-all-rspec` | Matches if the pipeline is for a merge request and the MR has label ~"pipeline:run-all-rspec". | | +| `if-merge-request-labels-run-cs-evaluation` | Matches if the pipeline is for a merge request and the MR has label ~"pipeline:run-CS-evaluation". | | | `if-security-merge-request` | Matches if the pipeline is for a security merge request. | | | `if-security-schedule` | Matches if the pipeline is for a security scheduled pipeline. | | | `if-nightly-master-schedule` | Matches if the pipeline is for a `master` scheduled pipeline with `$NIGHTLY` set. | | diff --git a/scripts/download-downstream-artifact.rb b/scripts/download-downstream-artifact.rb new file mode 100755 index 0000000000000000000000000000000000000000..23c400a9add16fbd7dfec41d0468c7012454b6b9 --- /dev/null +++ b/scripts/download-downstream-artifact.rb @@ -0,0 +1,121 @@ +#!/usr/bin/env ruby + +# frozen_string_literal: true + +require 'gitlab' + +require_relative 'api/default_options' + +# This class allows an upstream job to fetch an artifact from a job in a downstream pipeline. +# +# Until https://gitlab.com/gitlab-org/gitlab/-/issues/285100 is resolved it's not straightforward for an upstream +# pipeline to use artifacts from a downstream pipeline. There is a workaround for parent-child pipelines (see the issue) +# but it relies on CI_MERGE_REQUEST_REF_PATH so it doesn't work for multi-project pipelines. +# +# This uses the Jobs API to get pipeline bridges (trigger jobs) and the Job artifacts API to download artifacts. +# - https://docs.gitlab.com/ee/api/jobs.html#list-pipeline-trigger-jobs +# - https://docs.gitlab.com/ee/api/job_artifacts.html +# +# Note: This class also works for parent-child pipelines within the same project, it's just not necessary in that case. +class DownloadDownstreamArtifact + def initialize(options) + @upstream_project = options.fetch(:upstream_project, API::DEFAULT_OPTIONS[:project]) + @upstream_pipeline_id = options.fetch(:upstream_pipeline_id, API::DEFAULT_OPTIONS[:pipeline_id]) + @downstream_project = options.fetch(:downstream_project, API::DEFAULT_OPTIONS[:project]) + @downstream_job_name = options.fetch(:downstream_job_name) + @trigger_job_name = options.fetch(:trigger_job_name) + @downstream_artifact_path = options.fetch(:downstream_artifact_path) + @output_artifact_path = options.fetch(:output_artifact_path) + + unless options.key?(:api_token) + raise ArgumentError, 'PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE is required to access downstream pipelines' + end + + api_token = options.fetch(:api_token) + + @client = Gitlab.client( + endpoint: options.fetch(:endpoint, API::DEFAULT_OPTIONS[:endpoint]), + private_token: api_token + ) + end + + def execute + unless downstream_pipeline + abort("Could not find downstream pipeline triggered via #{trigger_job_name} in project #{downstream_project}") + end + + unless downstream_job + abort("Could not find job with name '#{downstream_job_name}' in #{downstream_pipeline['web_url']}") + end + + puts "Fetching scores artifact from downstream pipeline triggered via #{trigger_job_name}..." + puts "Downstream pipeline is #{downstream_pipeline['web_url']}." + puts %(Downstream job "#{downstream_job_name}": #{downstream_job['web_url']}.) + + path = downstream_artifact_path.sub('DOWNSTREAM_JOB_ID', downstream_job.id.to_s) + puts %(Fetching artifact "#{path}" from #{downstream_job_name}...) + + download_and_save_artifact(path) + + puts "Artifact saved as #{output_artifact_path} ..." + end + + def self.options_from_env + API::DEFAULT_OPTIONS.merge({ + upstream_project: API::DEFAULT_OPTIONS[:project], + upstream_pipeline_id: API::DEFAULT_OPTIONS[:pipeline_id], + downstream_project: ENV.fetch('DOWNSTREAM_PROJECT', API::DEFAULT_OPTIONS[:project]), + downstream_job_name: ENV['DOWNSTREAM_JOB_NAME'], + trigger_job_name: ENV['TRIGGER_JOB_NAME'], + downstream_artifact_path: ENV['DOWNSTREAM_JOB_ARTIFACT_PATH'], + output_artifact_path: ENV['OUTPUT_ARTIFACT_PATH'] + }).except(:project, :pipeline_id) + end + + private + + attr_reader :downstream_artifact_path, + :output_artifact_path, + :downstream_job_name, + :trigger_job_name, + :upstream_project, + :downstream_project, + :upstream_pipeline_id, + :client + + def bridge + @bridge ||= client + .pipeline_bridges(upstream_project, upstream_pipeline_id, per_page: 100) + .auto_paginate + .find { |job| job.name.include?(trigger_job_name) } + end + + def downstream_pipeline + @downstream_pipeline ||= + if bridge&.downstream_pipeline.nil? + nil + else + client.pipeline(downstream_project, bridge.downstream_pipeline.id) + end + end + + def downstream_job + @downstream_job ||= client + .pipeline_jobs(downstream_project, downstream_pipeline.id) + .find { |job| job.name.include?(downstream_job_name) } + end + + def download_and_save_artifact(job_artifact_path) + file_response = client.download_job_artifact_file(downstream_project, downstream_job.id, job_artifact_path) + + file_response.respond_to?(:read) || abort("Could not download artifact. Request returned: #{file_response}") + + File.write(output_artifact_path, file_response.read) + end +end + +if $PROGRAM_NAME == __FILE__ + options = DownloadDownstreamArtifact.options_from_env + + DownloadDownstreamArtifact.new(options).execute +end diff --git a/spec/scripts/download_downstream_artifact_spec.rb b/spec/scripts/download_downstream_artifact_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..05d1dc9933f0892aeaee5c3fa1342e9e4173f48c --- /dev/null +++ b/spec/scripts/download_downstream_artifact_spec.rb @@ -0,0 +1,180 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' +require 'gitlab/rspec/all' +require_relative '../../scripts/download-downstream-artifact' + +# rubocop:disable RSpec/VerifiedDoubles -- doubles are simple mocks of a few methods from external code + +RSpec.describe DownloadDownstreamArtifact, feature_category: :tooling do + include StubENV + + subject(:execute) { described_class.new(options).execute } + + before do + stub_env('PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE', nil) + stub_env('CI_PROJECT_ID', nil) + stub_env('CI_PIPELINE_ID', nil) + stub_env('CI_API_V4_URL', nil) + stub_env('DOWNSTREAM_PROJECT', nil) + stub_env('DOWNSTREAM_JOB_NAME', nil) + stub_env('TRIGGER_JOB_NAME', nil) + stub_env('DOWNSTREAM_JOB_ARTIFACT_PATH', nil) + stub_env('OUTPUT_ARTIFACT_PATH', nil) + end + + describe '#execute' do + let(:options) do + { + api_token: 'asdf1234', + endpoint: 'https://gitlab.com/api/v4', + upstream_project: 'upstream/project', + upstream_pipeline_id: 123, + downstream_project: 'downstream/project', + downstream_job_name: 'test-job', + trigger_job_name: 'trigger-job', + downstream_artifact_path: 'scores-DOWNSTREAM_JOB_ID.csv', + output_artifact_path: 'scores.csv' + } + end + + let(:client) { double('Gitlab::Client') } + let(:artifact_response) { double('io', read: 'artifact content') } + + let(:job) do + Struct.new(:id, :name, :web_url).new(789, 'test-job', 'https://example.com/jobs/789') + end + + let(:downstream_pipeline) do + Struct.new(:id, :web_url).new(111, 'https://example.com/pipelines/111') + end + + let(:pipeline_bridges) do + double('pipeline_bridges', auto_paginate: [double(name: 'trigger-job', downstream_pipeline: downstream_pipeline)]) + end + + let(:expected_output) do + <<~OUTPUT + Fetching scores artifact from downstream pipeline triggered via trigger-job... + Downstream pipeline is https://example.com/pipelines/111. + Downstream job "test-job": https://example.com/jobs/789. + Fetching artifact "scores-789.csv" from test-job... + Artifact saved as scores.csv ... + OUTPUT + end + + before do + allow(Gitlab).to receive(:client) + .with(endpoint: options[:endpoint], private_token: options[:api_token]) + .and_return(client) + + allow(client).to receive(:pipeline_bridges).and_return(pipeline_bridges) + allow(client).to receive(:pipeline).and_return(downstream_pipeline) + allow(client).to receive(:pipeline_jobs).and_return([job]) + allow(client).to receive(:download_job_artifact_file).and_return(artifact_response) + allow(File).to receive(:write) + end + + it 'downloads artifact from downstream pipeline' do + expect(client).to receive(:download_job_artifact_file).with('downstream/project', 789, 'scores-789.csv') + + expect { execute }.to output(expected_output).to_stdout + end + + it 'saves artifact to output path' do + expect(File).to receive(:write).with('scores.csv', 'artifact content') + + expect { execute }.to output(expected_output).to_stdout + end + + context 'when options come from environment variables' do + before do + stub_env('PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE', 'asdf1234') + stub_env('CI_PROJECT_ID', 'upstream/project') + stub_env('CI_PIPELINE_ID', '123') + stub_env('CI_API_V4_URL', 'https://gitlab.com/api/v4') + stub_env('DOWNSTREAM_PROJECT', 'downstream/project') + stub_env('DOWNSTREAM_JOB_NAME', 'test-job') + stub_env('TRIGGER_JOB_NAME', 'trigger-job') + stub_env('DOWNSTREAM_JOB_ARTIFACT_PATH', 'scores-DOWNSTREAM_JOB_ID.csv') + stub_env('OUTPUT_ARTIFACT_PATH', 'scores.csv') + + stub_const('API::DEFAULT_OPTIONS', { + project: ENV['CI_PROJECT_ID'], + pipeline_id: ENV['CI_PIPELINE_ID'], + api_token: ENV['PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE'], + endpoint: ENV['CI_API_V4_URL'] + }) + end + + it 'uses the environment variable values' do + options = described_class.options_from_env + + expect(File).to receive(:write) + expect { described_class.new(options).execute }.to output(expected_output).to_stdout + end + end + + context 'when the downstream pipeline cannot be found' do + let(:pipeline_bridges) do + double('pipeline_bridges', auto_paginate: [double(name: 'trigger-job', downstream_pipeline: nil)]) + end + + it 'aborts' do + expect(File).not_to receive(:write) + expect { described_class.new(options).execute } + .to output( + %r{Could not find downstream pipeline triggered via trigger-job in project downstream/project} + ).to_stderr + .and raise_error(SystemExit) + end + end + + context 'when the downstream job cannot be found' do + let(:job) { double('job', name: 'foo') } + + it 'aborts' do + expect(File).not_to receive(:write) + expect { described_class.new(options).execute } + .to output( + %r{Could not find job with name 'test-job' in https://example.com/pipelines/111} + ).to_stderr + .and raise_error(SystemExit) + end + end + + context 'when the downstream artifact cannot be found' do + let(:artifact_response) { 'error' } + + it 'aborts' do + expect(File).not_to receive(:write) + expect { described_class.new(options).execute } + .to output( + %r{Could not download artifact. Request returned: error} + ).to_stderr + .and raise_error(SystemExit) + end + end + end + + context 'when called without an API token' do + let(:options) do + { + endpoint: 'https://gitlab.com/api/v4', + upstream_project: 'upstream/project', + upstream_pipeline_id: 123, + downstream_project: 'downstream/project', + downstream_job_name: 'test-job', + trigger_job_name: 'trigger-job', + downstream_artifact_path: 'scores-DOWNSTREAM_JOB_ID.csv', + output_artifact_path: 'scores.csv' + } + end + + it 'raises an error' do + expect { described_class.new(options) }.to raise_error(ArgumentError) + end + end +end + +# rubocop:enable RSpec/VerifiedDoubles