Merge branch 'feature/runner-jobs-v4-api' into 'master'

Feature/runner jobs v4 api

Closes #28513

See merge request !9273

app/models/ci/build.rb

@@ -517,6 +517,27 @@ def user_variables
       ]
     end
 
+    def steps
+      [Gitlab::Ci::Build::Step.from_commands(self),
+       Gitlab::Ci::Build::Step.from_after_script(self)].compact
+    end
+
+    def image
+      Gitlab::Ci::Build::Image.from_image(self)
+    end
+
+    def services
+      Gitlab::Ci::Build::Image.from_services(self)
+    end
+
+    def artifacts
+      [options[:artifacts]]
+    end
+
+    def cache
+      [options[:cache]]
+    end
+
     def credentials
       Gitlab::Ci::Build::Credentials::Factory.new(self).create!
     end

app/services/ci/register_build_service.rb → app/services/ci/register_job_service.rb

 module Ci
   # This class responsible for assigning
   # proper pending build to runner on runner API request
-  class RegisterBuildService
+  class RegisterJobService
     include Gitlab::CurrentSettings
 
     attr_reader :runner

changelogs/unreleased/feature-runner-jobs-v4-api.yml

+---
+title: Add Runner's jobs v4 API
+merge_request: 9273
+author:

lib/api/entities.rb

@@ -705,5 +705,83 @@ class BroadcastMessage < Grape::Entity
       expose :id, :message, :starts_at, :ends_at, :color, :font
       expose :active?, as: :active
     end
+
+    module JobRequest
+      class JobInfo < Grape::Entity
+        expose :name, :stage
+        expose :project_id, :project_name
+      end
+
+      class GitInfo < Grape::Entity
+        expose :repo_url, :ref, :sha, :before_sha
+        expose :ref_type do |model|
+          if model.tag
+            'tag'
+          else
+            'branch'
+          end
+        end
+      end
+
+      class RunnerInfo < Grape::Entity
+        expose :timeout
+      end
+
+      class Step < Grape::Entity
+        expose :name, :script, :timeout, :when, :allow_failure
+      end
+
+      class Image < Grape::Entity
+        expose :name
+      end
+
+      class Artifacts < Grape::Entity
+        expose :name, :untracked, :paths, :when, :expire_in
+      end
+
+      class Cache < Grape::Entity
+        expose :key, :untracked, :paths
+      end
+
+      class Credentials < Grape::Entity
+        expose :type, :url, :username, :password
+      end
+
+      class ArtifactFile < Grape::Entity
+        expose :filename, :size
+      end
+
+      class Dependency < Grape::Entity
+        expose :id, :name
+        expose :artifacts_file, using: ArtifactFile, if: ->(job, _) { job.artifacts? }
+      end
+
+      class Response < Grape::Entity
+        expose :id
+        expose :token
+        expose :allow_git_fetch
+
+        expose :job_info, using: JobInfo do |model|
+          model
+        end
+
+        expose :git_info, using: GitInfo do |model|
+          model
+        end
+
+        expose :runner_info, using: RunnerInfo do |model|
+          model
+        end
+
+        expose :variables
+        expose :steps, using: Step
+        expose :image, using: Image
+        expose :services, using: Image
+        expose :artifacts, using: Artifacts
+        expose :cache, using: Cache
+        expose :credentials, using: Credentials
+        expose :depends_on_builds, as: :dependencies, using: Dependency
+      end
+    end
   end
 end

lib/api/helpers/runner.rb

 module API
   module Helpers
     module Runner
+      JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN'.freeze
+      JOB_TOKEN_PARAM = :token
+      UPDATE_RUNNER_EVERY = 10 * 60
+
       def runner_registration_token_valid?
         ActiveSupport::SecurityUtils.variable_size_secure_compare(params[:token],
                                                                   current_application_settings.runners_registration_token)
@@ -18,6 +22,56 @@ def authenticate_runner!
       def current_runner
         @runner ||= ::Ci::Runner.find_by_token(params[:token].to_s)
       end
+
+      def update_runner_info
+        return unless update_runner?
+
+        current_runner.contacted_at = Time.now
+        current_runner.assign_attributes(get_runner_version_from_params)
+        current_runner.save if current_runner.changed?
+      end
+
+      def update_runner?
+        # Use a random threshold to prevent beating DB updates.
+        # It generates a distribution between [40m, 80m].
+        #
+        contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY)
+
+        current_runner.contacted_at.nil? ||
+          (Time.now - current_runner.contacted_at) >= contacted_at_max_age
+      end
+
+      def job_not_found!
+        if headers['User-Agent'].to_s =~ /gitlab(-ci-multi)?-runner \d+\.\d+\.\d+(~beta\.\d+\.g[0-9a-f]+)? /
+          no_content!
+        else
+          not_found!
+        end
+      end
+
+      def validate_job!(job)
+        not_found! unless job
+
+        yield if block_given?
+
+        forbidden!('Project has been deleted!') unless job.project
+        forbidden!('Job has been erased!') if job.erased?
+      end
+
+      def authenticate_job!(job)
+        validate_job!(job) do
+          forbidden! unless job_token_valid?(job)
+        end
+      end
+
+      def job_token_valid?(job)
+        token = (params[JOB_TOKEN_PARAM] || env[JOB_TOKEN_HEADER]).to_s
+        token && job.valid_token?(token)
+      end
+
+      def max_artifacts_size
+        current_application_settings.max_artifacts_size.megabytes.to_i
+      end
     end
   end
 end

lib/api/runner.rb

@@ -48,5 +48,203 @@ class Runner < Grape::API
         Ci::Runner.find_by_token(params[:token]).destroy
       end
     end
+
+    resource :jobs do
+      desc 'Request a job' do
+        success Entities::JobRequest::Response
+      end
+      params do
+        requires :token, type: String, desc: %q(Runner's authentication token)
+        optional :last_update, type: String, desc: %q(Runner's queue last_update token)
+        optional :info, type: Hash, desc: %q(Runner's metadata)
+      end
+      post '/request' do
+        authenticate_runner!
+        not_found! unless current_runner.active?
+        update_runner_info
+
+        if current_runner.is_runner_queue_value_latest?(params[:last_update])
+          header 'X-GitLab-Last-Update', params[:last_update]
+          Gitlab::Metrics.add_event(:build_not_found_cached)
+          return job_not_found!
+        end
+
+        new_update = current_runner.ensure_runner_queue_value
+        result = ::Ci::RegisterJobService.new(current_runner).execute
+
+        if result.valid?
+          if result.build
+            Gitlab::Metrics.add_event(:build_found,
+                                      project: result.build.project.path_with_namespace)
+            present result.build, with: Entities::JobRequest::Response
+          else
+            Gitlab::Metrics.add_event(:build_not_found)
+            header 'X-GitLab-Last-Update', new_update
+            job_not_found!
+          end
+        else
+          # We received build that is invalid due to concurrency conflict
+          Gitlab::Metrics.add_event(:build_invalid)
+          conflict!
+        end
+      end
+
+      desc 'Updates a job' do
+        http_codes [[200, 'Job was updated'], [403, 'Forbidden']]
+      end
+      params do
+        requires :token, type: String, desc: %q(Runners's authentication token)
+        requires :id, type: Integer, desc: %q(Job's ID)
+        optional :trace, type: String, desc: %q(Job's full trace)
+        optional :state, type: String, desc: %q(Job's status: success, failed)
+      end
+      put '/:id' do
+        job = Ci::Build.find_by_id(params[:id])
+        authenticate_job!(job)
+
+        job.update_attributes(trace: params[:trace]) if params[:trace]
+
+        Gitlab::Metrics.add_event(:update_build,
+                                  project: job.project.path_with_namespace)
+
+        case params[:state].to_s
+        when 'success'
+          job.success
+        when 'failed'
+          job.drop
+        end
+      end
+
+      desc 'Appends a patch to the job trace' do
+        http_codes [[202, 'Trace was patched'],
+                    [400, 'Missing Content-Range header'],
+                    [403, 'Forbidden'],
+                    [416, 'Range not satisfiable']]
+      end
+      params do
+        requires :id, type: Integer, desc: %q(Job's ID)
+        optional :token, type: String, desc: %q(Job's authentication token)
+      end
+      patch '/:id/trace' do
+        job = Ci::Build.find_by_id(params[:id])
+        authenticate_job!(job)
+
+        error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range')
+        content_range = request.headers['Content-Range']
+        content_range = content_range.split('-')
+
+        current_length = job.trace_length
+        unless current_length == content_range[0].to_i
+          return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{current_length}" })
+        end
+
+        job.append_trace(request.body.read, content_range[0].to_i)
+
+        status 202
+        header 'Job-Status', job.status
+        header 'Range', "0-#{job.trace_length}"
+      end
+
+      desc 'Authorize artifacts uploading for job' do
+        http_codes [[200, 'Upload allowed'],
+                    [403, 'Forbidden'],
+                    [405, 'Artifacts support not enabled'],
+                    [413, 'File too large']]
+      end
+      params do
+        requires :id, type: Integer, desc: %q(Job's ID)
+        optional :token, type: String, desc: %q(Job's authentication token)
+        optional :filesize, type: Integer, desc: %q(Artifacts filesize)
+      end
+      post '/:id/artifacts/authorize' do
+        not_allowed! unless Gitlab.config.artifacts.enabled
+        require_gitlab_workhorse!
+        Gitlab::Workhorse.verify_api_request!(headers)
+
+        job = Ci::Build.find_by_id(params[:id])
+        authenticate_job!(job)
+        forbidden!('Job is not running') unless job.running?
+
+        if params[:filesize]
+          file_size = params[:filesize].to_i
+          file_to_large! unless file_size < max_artifacts_size
+        end
+
+        status 200
+        content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
+        Gitlab::Workhorse.artifact_upload_ok
+      end
+
+      desc 'Upload artifacts for job' do
+        success Entities::JobRequest::Response
+        http_codes [[201, 'Artifact uploaded'],
+                    [400, 'Bad request'],
+                    [403, 'Forbidden'],
+                    [405, 'Artifacts support not enabled'],
+                    [413, 'File too large']]
+      end
+      params do
+        requires :id, type: Integer, desc: %q(Job's ID)
+        optional :token, type: String, desc: %q(Job's authentication token)
+        optional :expire_in, type: String, desc: %q(Specify when artifacts should expire)
+        optional :file, type: File, desc: %q(Artifact's file)
+        optional 'file.path', type: String, desc: %q(path to locally stored body (generated by Workhorse))
+        optional 'file.name', type: String, desc: %q(real filename as send in Content-Disposition (generated by Workhorse))
+        optional 'file.type', type: String, desc: %q(real content type as send in Content-Type (generated by Workhorse))
+        optional 'metadata.path', type: String, desc: %q(path to locally stored body (generated by Workhorse))
+        optional 'metadata.name', type: String, desc: %q(filename (generated by Workhorse))
+      end
+      post '/:id/artifacts' do
+        not_allowed! unless Gitlab.config.artifacts.enabled
+        require_gitlab_workhorse!
+
+        job = Ci::Build.find_by_id(params[:id])
+        authenticate_job!(job)
+        forbidden!('Job is not running!') unless job.running?
+
+        artifacts_upload_path = ArtifactUploader.artifacts_upload_path
+        artifacts = uploaded_file(:file, artifacts_upload_path)
+        metadata = uploaded_file(:metadata, artifacts_upload_path)
+
+        bad_request!('Missing artifacts file!') unless artifacts
+        file_to_large! unless artifacts.size < max_artifacts_size
+
+        job.artifacts_file = artifacts
+        job.artifacts_metadata = metadata
+        job.artifacts_expire_in = params['expire_in'] ||
+          Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in
+
+        if job.save
+          present job, with: Entities::JobRequest::Response
+        else
+          render_validation_error!(job)
+        end
+      end
+
+      desc 'Download the artifacts file for job' do
+        http_codes [[200, 'Upload allowed'],
+                    [403, 'Forbidden'],
+                    [404, 'Artifact not found']]
+      end
+      params do
+        requires :id, type: Integer, desc: %q(Job's ID)
+        optional :token, type: String, desc: %q(Job's authentication token)
+      end
+      get '/:id/artifacts' do
+        job = Ci::Build.find_by_id(params[:id])
+        authenticate_job!(job)
+
+        artifacts_file = job.artifacts_file
+        unless artifacts_file.file_storage?
+          return redirect_to job.artifacts_file.url
+        end
+
+        unless artifacts_file.exists?
+          not_found!
+        end
+
+        present_file!(artifacts_file.path, artifacts_file.filename)
+      end
+    end
   end
 end

lib/ci/api/builds.rb

@@ -24,7 +24,7 @@ class Builds < Grape::API
 
           new_update = current_runner.ensure_runner_queue_value
 
-          result = Ci::RegisterBuildService.new(current_runner).execute
+          result = Ci::RegisterJobService.new(current_runner).execute
 
           if result.valid?
             if result.build

lib/gitlab/ci/build/image.rb

+module Gitlab
+  module Ci
+    module Build
+      class Image
+        attr_reader :name
+
+        class << self
+          def from_image(job)
+            image = Gitlab::Ci::Build::Image.new(job.options[:image])
+            return unless image.valid?
+            image
+          end
+
+          def from_services(job)
+            services = job.options[:services].to_a.map do |service|
+              Gitlab::Ci::Build::Image.new(service)
+            end
+
+            services.select(&:valid?).compact
+          end
+        end
+
+        def initialize(image)
+          @name = image
+        end
+
+        def valid?
+          @name.present?
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/build/step.rb

+module Gitlab
+  module Ci
+    module Build
+      class Step
+        WHEN_ON_FAILURE = 'on_failure'.freeze
+        WHEN_ON_SUCCESS = 'on_success'.freeze
+        WHEN_ALWAYS = 'always'.freeze
+
+        attr_reader :name
+        attr_writer :script
+        attr_accessor :timeout, :when, :allow_failure
+
+        class << self
+          def from_commands(job)
+            self.new(:script).tap do |step|
+              step.script = job.commands
+              step.timeout = job.timeout
+              step.when = WHEN_ON_SUCCESS
+            end
+          end
+
+          def from_after_script(job)
+            after_script = job.options[:after_script]
+            return unless after_script
+
+            self.new(:after_script).tap do |step|
+              step.script = after_script
+              step.timeout = job.timeout
+              step.when = WHEN_ALWAYS
+              step.allow_failure = true
+            end
+          end
+        end
+
+        def initialize(name)
+          @name = name
+          @allow_failure = false
+        end
+
+        def script
+          @script.split("\n")
+        end
+      end
+    end
+  end
+end

spec/factories/ci/builds.rb

@@ -15,8 +15,8 @@
 
     options do
       {
-        image: "ruby:2.1",
-        services: ["postgres"]
+        image: 'ruby:2.1',
+        services: ['postgres']
       }
     end
 
@@ -166,5 +166,31 @@
         allow(build).to receive(:commit).and_return build(:commit)
       end
     end
+
+    trait :extended_options do
+      options do
+        {
+            image: 'ruby:2.1',
+            services: ['postgres'],
+            after_script: "ls\ndate",
+            artifacts: {
+                name: 'artifacts_file',
+                untracked: false,
+                paths: ['out/'],
+                when: 'always',
+                expire_in: '7d'
+            },
+            cache: {
+                key: 'cache_key',
+                untracked: false,
+                paths: ['vendor/*']
+            }
+        }
+      end
+    end
+
+    trait :no_options do
+      options { {} }
+    end
   end
 end

spec/lib/gitlab/ci/build/image_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Build::Image do
+  let(:job) { create(:ci_build, :no_options) }
+
+  describe '#from_image' do
+    subject { described_class.from_image(job) }
+
+    context 'when image is defined in job' do
+      let(:image_name) { 'ruby:2.1' }
+      let(:job) { create(:ci_build, options: { image: image_name } ) }
+
+      it 'fabricates an object of the proper class' do
+        is_expected.to be_kind_of(described_class)
+      end
+
+      it 'populates fabricated object with the proper name attribute' do
+        expect(subject.name).to eq(image_name)
+      end
+
+      context 'when image name is empty' do
+        let(:image_name) { '' }
+
+        it 'does not fabricate an object' do
+          is_expected.to be_nil
+        end
+      end
+    end
+
+    context 'when image is not defined in job' do
+      it 'does not fabricate an object' do
+        is_expected.to be_nil
+      end
+    end
+  end
+
+  describe '#from_services' do
+    subject { described_class.from_services(job) }
+
+    context 'when services are defined in job' do
+      let(:service_image_name) { 'postgres' }
+      let(:job) { create(:ci_build, options: { services: [service_image_name] }) }
+
+      it 'fabricates an non-empty array of objects' do
+        is_expected.to be_kind_of(Array)
+        is_expected.not_to be_empty
+        expect(subject.first.name).to eq(service_image_name)
+      end
+
+      context 'when service image name is empty' do
+        let(:service_image_name) { '' }
+
+        it 'fabricates an empty array' do
+          is_expected.to be_kind_of(Array)
+          is_expected.to be_empty
+        end
+      end
+    end
+
+    context 'when services are not defined in job' do
+      it 'fabricates an empty array' do
+        is_expected.to be_kind_of(Array)
+        is_expected.to be_empty
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/build/step_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Build::Step do
+  let(:job) { create(:ci_build, :no_options, commands: "ls -la\ndate") }
+
+  describe '#from_commands' do
+    subject { described_class.from_commands(job) }
+
+    it 'fabricates an object' do
+      expect(subject.name).to eq(:script)
+      expect(subject.script).to eq(['ls -la', 'date'])
+      expect(subject.timeout).to eq(job.timeout)
+      expect(subject.when).to eq('on_success')
+      expect(subject.allow_failure).to be_falsey
+    end
+  end
+
+  describe '#from_after_script' do
+    subject { described_class.from_after_script(job) }
+
+    context 'when after_script is empty' do
+      it 'doesn not fabricate an object' do
+        is_expected.to be_nil
+      end
+    end
+
+    context 'when after_script is not empty' do
+      let(:job) { create(:ci_build, options: { after_script: "ls -la\ndate" }) }
+
+      it 'fabricates an object' do
+        expect(subject.name).to eq(:after_script)
+        expect(subject.script).to eq(['ls -la', 'date'])
+        expect(subject.timeout).to eq(job.timeout)
+        expect(subject.when).to eq('always')
+        expect(subject.allow_failure).to be_truthy
+      end
+    end
+  end
+end

spec/services/ci/register_build_service_spec.rb → spec/services/ci/register_job_service_spec.rb

 require 'spec_helper'
 
 module Ci
-  describe RegisterBuildService, services: true do
+  describe RegisterJobService, services: true do
     let!(:project) { FactoryGirl.create :empty_project, shared_runners_enabled: false }
     let!(:pipeline) { FactoryGirl.create :ci_pipeline, project: project }
     let!(:pending_build) { FactoryGirl.create :ci_build, pipeline: pipeline }
@@ -181,7 +181,7 @@ module Ci
           let!(:other_build) { create :ci_build, pipeline: pipeline }
 
           before do
-            allow_any_instance_of(Ci::RegisterBuildService).to receive(:builds_for_specific_runner)
+            allow_any_instance_of(Ci::RegisterJobService).to receive(:builds_for_specific_runner)
               .and_return([pending_build, other_build])
           end
 
@@ -193,7 +193,7 @@ module Ci
 
         context 'when single build is in queue' do
           before do
-            allow_any_instance_of(Ci::RegisterBuildService).to receive(:builds_for_specific_runner)
+            allow_any_instance_of(Ci::RegisterJobService).to receive(:builds_for_specific_runner)
               .and_return([pending_build])
           end
 
@@ -204,7 +204,7 @@ module Ci
 
         context 'when there is no build in queue' do
           before do
-            allow_any_instance_of(Ci::RegisterBuildService).to receive(:builds_for_specific_runner)
+            allow_any_instance_of(Ci::RegisterJobService).to receive(:builds_for_specific_runner)
               .and_return([])
           end