Merge branch 'fp-fetch-spec-from-ci-config' into 'master'

Add service that fetches expected inputs for a pipeline

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/181996



Merged-by: Fabio Pitino <fpitino@gitlab.com>
Approved-by: Marius Bobin <mbobin@gitlab.com>
Approved-by: Furkan Ayhan <furkanayhn@gmail.com>
Reviewed-by: Furkan Ayhan <furkanayhn@gmail.com>
Reviewed-by: Fabio Pitino <fpitino@gitlab.com>
Reviewed-by: Marius Bobin <mbobin@gitlab.com>

app/services/ci/pipeline_creation/find_ci_config_spec_service.rb

+# frozen_string_literal: true
+
+module Ci
+  module PipelineCreation
+    class FindCiConfigSpecService
+      include Gitlab::Utils::StrongMemoize
+
+      # This service is used by the frontend to display inputs as an HTML form
+      # when creating a pipeline as a web request.
+      # For the reason we are defaulting `pipeline_source` to be `web`.
+      def initialize(current_user:, project:, ref:, pipeline_source: :web)
+        @current_user = current_user
+        @project = project
+        @ref = ref
+        @pipeline_source = pipeline_source
+      end
+
+      def execute
+        unless current_user.can?(:download_code, project)
+          return error_response('insufficient permissions to read inputs')
+        end
+
+        if !project.repository.branch_or_tag?(ref) || sha.blank?
+          return error_response('ref can only be an existing branch or tag')
+        end
+
+        return error_response('config not found') unless project_config.exists?
+
+        # Since CI Config path is configurable (local, other project, URL) we translate
+        # all supported config types into an `include: {...}` statement.
+        # The inputs we are looking for are not directly defined at this level of YAML
+        # but inside the included file.
+        if project_config.internal_include_prepended?
+          # We need to read the uninterpolated YAML of the included file.
+          yaml_content = ::Gitlab::Ci::Config::Yaml.load!(project_config.content)
+          yaml_result = yaml_result_of_internal_include(yaml_content)
+
+          return error_response('invalid YAML config') unless yaml_result&.valid?
+
+          success_response(yaml_result.spec)
+        else
+          # For now we do nothing. The unsupported case is `ProjectConfig::SecurityPolicyDefault`
+          # which is used when the project has no CI config explicitly defined but it's enforced
+          # by default using policies.
+          success_response({})
+        end
+      rescue ::Gitlab::Ci::Config::Yaml::LoadError => e
+        error_response("YAML load error: #{e.message}")
+      end
+
+      private
+
+      attr_reader :current_user, :project, :ref, :pipeline_source
+
+      def success_response(spec)
+        ServiceResponse.success(payload: { spec: spec })
+      end
+
+      def error_response(message)
+        ServiceResponse.error(message: message)
+      end
+
+      def project_config
+        ::Gitlab::Ci::ProjectConfig.new(project: project, ref: ref, sha: sha, pipeline_source: pipeline_source)
+      end
+      strong_memoize_attr :project_config
+
+      # TODO: temporary technical debt until https://gitlab.com/gitlab-org/gitlab/-/issues/520828
+      def yaml_result_of_internal_include(content)
+        locations = content[:include]
+        return if locations.blank?
+
+        files = ::Gitlab::Ci::Config::External::Mapper::Matcher.new(context).process(locations)
+
+        ::Gitlab::Ci::Config::External::Mapper::Verifier.new(context).skip_load_content!.process(files)
+
+        files.first&.load_uninterpolated_yaml
+      end
+
+      def context
+        ::Gitlab::Ci::Config::External::Context.new(
+          project: project,
+          sha: sha,
+          user: current_user)
+      end
+      strong_memoize_attr :context
+
+      def sha
+        project.commit(ref)&.sha
+      end
+      strong_memoize_attr :sha
+    end
+  end
+end

lib/gitlab/ci/config/external/file/base.rb

@@ -104,6 +104,10 @@ def load_and_validate_expanded_hash!
               validate_hash!
             end
 
+            def load_uninterpolated_yaml
+              ::Gitlab::Ci::Config::Yaml::Loader.new(content).load_uninterpolated_yaml
+            end
+
             protected
 
             def content_inputs

lib/gitlab/ci/config/external/mapper/verifier.rb

@@ -9,6 +9,11 @@ module External
         class Mapper
           # Fetches file contents and verifies them
           class Verifier < Base
+            # TODO: remove with https://gitlab.com/gitlab-org/gitlab/-/issues/520828
+            def skip_load_content!
+              tap { @skip_load_content = true }
+            end
+
             private
 
             def process_without_instrumentation(files)
@@ -38,7 +43,8 @@ def process_without_instrumentation(files)
                 verify_execution_time!
 
                 file.validate_content! if file.valid?
-                file.load_and_validate_expanded_hash! if file.valid?
+
+                file.load_and_validate_expanded_hash! if file.valid? && !@skip_load_content
 
                 next unless file.valid?
 

spec/services/ci/pipeline_creation/find_ci_config_spec_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Ci::PipelineCreation::FindCiConfigSpecService, feature_category: :pipeline_composition do
+  let(:project) { create(:project, :repository) }
+  let_it_be(:user) { create(:user) }
+
+  let(:ref) { 'master' }
+  let(:pipeline_source) { :web }
+
+  subject(:service) do
+    described_class.new(current_user: user, project: project, ref: ref, pipeline_source: pipeline_source)
+  end
+
+  describe '#execute' do
+    let(:config_yaml_without_inputs) do
+      <<~YAML
+      job:
+        script: echo hello world
+      YAML
+    end
+
+    let(:config_yaml) do
+      <<~YAML
+      spec:
+        inputs:
+          foo:
+            default: bar
+      ---
+      job:
+        script: echo hello world
+      YAML
+    end
+
+    let(:expected_spec) do
+      { inputs: { foo: { default: 'bar' } } }
+    end
+
+    shared_examples 'successful response without spec' do
+      let(:config_yaml) { config_yaml_without_inputs }
+
+      it 'returns success response without spec' do
+        result = service.execute
+
+        expect(result).to be_success
+        expect(result.payload).to eq(spec: {})
+      end
+    end
+
+    shared_examples 'successful response with spec' do
+      it 'returns success response with spec' do
+        result = service.execute
+
+        expect(result).to be_success
+        expect(result.payload).to eq(spec: expected_spec)
+      end
+    end
+
+    context 'when user does not have permission to read code' do
+      before do
+        project.add_guest(user)
+      end
+
+      it 'returns error response' do
+        result = service.execute
+
+        expect(result).to be_error
+        expect(result.message).to eq('insufficient permissions to read inputs')
+      end
+    end
+
+    context 'when user has permissions to read code' do
+      before do
+        project.add_developer(user)
+      end
+
+      context 'when ref does not exist' do
+        let(:ref) { 'non-existent-branch' }
+
+        it 'returns error response' do
+          result = service.execute
+
+          expect(result).to be_error
+          expect(result.message).to eq('ref can only be an existing branch or tag')
+        end
+      end
+
+      context 'when ref is a SHA' do
+        let(:ref) { project.commit('master')&.sha }
+
+        it 'returns error response' do
+          result = service.execute
+
+          expect(result).to be_error
+          expect(result.message).to eq('ref can only be an existing branch or tag')
+        end
+      end
+
+      context 'when config does not exist and AutoDevOps is disabled' do
+        before do
+          allow(project).to receive(:auto_devops_enabled?).and_return(false)
+        end
+
+        it 'returns error response' do
+          result = service.execute
+
+          expect(result).to be_error
+          expect(result.message).to eq('config not found')
+        end
+      end
+
+      context 'when config is expected in the project' do
+        before do
+          project.repository.create_file(
+            project.creator,
+            '.gitlab-ci.yml',
+            config_yaml,
+            message: 'Add CI',
+            branch_name: 'master')
+        end
+
+        it_behaves_like 'successful response with spec'
+        it_behaves_like 'successful response without spec'
+
+        context 'when an error occurs during yaml processing' do
+          let(:config_yaml) do
+            <<~YAML
+            a*
+            test: <<a
+            YAML
+          end
+
+          it 'returns error response' do
+            result = service.execute
+
+            expect(result).to be_error
+            expect(result.message).to eq('invalid YAML config')
+          end
+        end
+
+        context 'when an error occurs during yaml loading' do
+          it 'returns error response' do
+            allow(::Gitlab::Ci::Config::Yaml)
+              .to receive(:load!)
+              .and_raise(::Gitlab::Ci::Config::Yaml::LoadError)
+
+            result = service.execute
+
+            expect(result).to be_error
+            expect(result.message).to match(/YAML load error/)
+          end
+        end
+      end
+
+      context 'when config is expected on another project' do
+        let!(:another_project) { create(:project, :repository) }
+
+        before do
+          another_project.add_developer(user)
+          another_project.repository.create_file(
+            another_project.creator,
+            'config.yml',
+            config_yaml,
+            message: 'Add CI',
+            branch_name: 'master')
+
+          project.update!(ci_config_path: "config.yml@#{another_project.full_path}")
+        end
+
+        it_behaves_like 'successful response with spec'
+        it_behaves_like 'successful response without spec'
+      end
+
+      context 'when config exists without internal include' do
+        before do
+          allow_next_instance_of(Gitlab::Ci::ProjectConfig) do |config|
+            allow(config).to receive_messages(exists?: true, internal_include_prepended?: false)
+          end
+        end
+
+        it 'returns success response with empty spec' do
+          result = service.execute
+
+          expect(result).to be_success
+          expect(result.payload).to eq({ spec: {} })
+        end
+      end
+    end
+  end
+end