From aea4041ce96f18afea70da15af3cbe1be4fa1f94 Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Wed, 18 May 2016 15:21:51 -0500
Subject: [PATCH] Allow to expire build artifacts
---
Gemfile | 3 +++
Gemfile.lock | 4 ++++
app/controllers/projects/builds_controller.rb | 6 ++++++
app/models/ci/build.rb | 18 ++++++++++++++++--
app/views/projects/builds/_sidebar.html.haml | 9 +++++++++
app/workers/expire_build_artifacts.rb | 12 ++++++++++++
config/gitlab.yml.example | 3 +++
config/initializers/1_settings.rb | 3 +++
config/routes.rb | 1 +
...1_add_artifacts_expire_date_to_ci_builds.rb | 5 +++++
lib/ci/api/builds.rb | 2 ++
lib/ci/gitlab_ci_yaml_processor.rb | 2 +-
12 files changed, 65 insertions(+), 3 deletions(-)
create mode 100644 app/workers/expire_build_artifacts.rb
create mode 100644 db/migrate/20160518200441_add_artifacts_expire_date_to_ci_builds.rb
diff --git a/Gemfile b/Gemfile
index b2660144f2b76..f56daa099a275 100644
--- a/Gemfile
+++ b/Gemfile
@@ -210,6 +210,9 @@ gem 'mousetrap-rails', '~> 1.4.6'
# Detect and convert string character encoding
gem 'charlock_holmes', '~> 0.7.3'
+# Parse duration
+gem 'chronic_duration', '~> 0.10.6'
+
gem "sass-rails", '~> 5.0.0'
gem "coffee-rails", '~> 4.1.0'
gem "uglifier", '~> 2.7.2'
diff --git a/Gemfile.lock b/Gemfile.lock
index dfc1570049459..2b2e2d2bb0773 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -133,6 +133,8 @@ GEM
mime-types (>= 1.16)
cause (0.1)
charlock_holmes (0.7.3)
+ chronic_duration (0.10.6)
+ numerizer (~> 0.1.1)
chunky_png (1.3.5)
cliver (0.3.2)
coderay (1.1.0)
@@ -424,6 +426,7 @@ GEM
nokogiri (1.6.8)
mini_portile2 (~> 2.1.0)
pkg-config (~> 1.1.7)
+ numerizer (0.1.1)
oauth (0.4.7)
oauth2 (1.0.0)
faraday (>= 0.8, < 0.10)
@@ -857,6 +860,7 @@ DEPENDENCIES
capybara-screenshot (~> 1.0.0)
carrierwave (~> 0.10.0)
charlock_holmes (~> 0.7.3)
+ chronic_duration (~> 0.10.6)
coffee-rails (~> 4.1.0)
connection_pool (~> 2.0)
coveralls (~> 0.8.2)
diff --git a/app/controllers/projects/builds_controller.rb b/app/controllers/projects/builds_controller.rb
index 14c828263428e..514f1b507fef9 100644
--- a/app/controllers/projects/builds_controller.rb
+++ b/app/controllers/projects/builds_controller.rb
@@ -78,6 +78,12 @@ def raw
end
end
+ def keep_artifacts
+ @build.keep_artifacts
+ redirect_to namespace_project_build_path(project.namespace, project, @build),
+ notice: "Artifacts will not be removed!"
+ end
+
private
def build
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index 6a64ca451f77f..74084b650cfb8 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -11,6 +11,8 @@ class Build < CommitStatus
scope :unstarted, ->() { where(runner_id: nil) }
scope :ignore_failures, ->() { where(allow_failure: false) }
+ scope :with_artifacts, ->() { where.not(artifacts_file: nil) }
+ scope :with_artifacts_expired, ->() { with_artifacts.where('artifacts_expire_at < ?', Time.now) }
mount_uploader :artifacts_file, ArtifactUploader
mount_uploader :artifacts_metadata, ArtifactUploader
@@ -328,11 +330,15 @@ def artifacts_metadata_entry(path, **options)
Gitlab::Ci::Build::Artifacts::Metadata.new(artifacts_metadata.path, path, **options).to_entry
end
+ def erase_artifacts!
+ remove_artifacts_file!
+ remove_artifacts_metadata!
+ end
+
def erase(opts = {})
return false unless erasable?
- remove_artifacts_file!
- remove_artifacts_metadata!
+ erase_artifacts!
erase_trace!
update_erased!(opts[:erased_by])
end
@@ -345,6 +351,14 @@ def erased?
!self.erased_at.nil?
end
+ def artifacts_expired?
+ self.artifacts_expire_at < Time.now && !artifacts?
+ end
+
+ def keep_artifacts
+ self.update(artifacts_expire_at: nil)
+ end
+
private
def erase_trace!
diff --git a/app/views/projects/builds/_sidebar.html.haml b/app/views/projects/builds/_sidebar.html.haml
index 5d931389dfbf5..d1a0da29ef762 100644
--- a/app/views/projects/builds/_sidebar.html.haml
+++ b/app/views/projects/builds/_sidebar.html.haml
@@ -44,6 +44,15 @@
%p.build-detail-row
%span.build-light-text Erased:
#{time_ago_with_tooltip(@build.erased_at)}
+ - elsif @build.artifacts_expired?
+ %p.build-detail-row.artifacts-expired.alert.alert-warning
+ The artifacts were removed #{time_ago_with_tooltip(@build.artifacts_expire_at)}
+ - elsif @build.artifacts_expire_at
+ %p.build-detail-row.artifacts-expired.alert.alert-info
+ The artifacts will be removed at #{time_ago_with_tooltip(@build.artifacts_expire_at)}
+ .pull-right
+ = link_to keep_artifacts_namespace_project_build_artifacts_path(@project.namespace, @project, @build), class: 'btn btn-sm btn-primary' do
+ Keep
%p.build-detail-row
%span.build-light-text Runner:
- if @build.runner && current_user && current_user.admin
diff --git a/app/workers/expire_build_artifacts.rb b/app/workers/expire_build_artifacts.rb
new file mode 100644
index 0000000000000..3d809d8ab6b0d
--- /dev/null
+++ b/app/workers/expire_build_artifacts.rb
@@ -0,0 +1,12 @@
+class ExpireBuildArtifacts
+ include Sidekiq::Worker
+
+ def perform
+ Rails.logger.info 'Cleaning old build artifacts'
+
+ builds = Ci::Build.with_artifacts_expired
+ builds.find_each(batch_size: 50).each do |build|
+ build.erase_artifacts!
+ end
+ end
+end
diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example
index 1048ef6e243bf..7b37e92ed46ff 100644
--- a/config/gitlab.yml.example
+++ b/config/gitlab.yml.example
@@ -164,6 +164,9 @@ production: &base
# Flag stuck CI builds as failed
stuck_ci_builds_worker:
cron: "0 0 * * *"
+ # Remove old artifacts
+ expire_build_artifacts:
+ cron: "50 * * * *"
# Periodically run 'git fsck' on all repositories. If started more than
# once per hour you will have concurrent 'git fsck' jobs.
repository_check_worker:
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index 436751b9d168f..b412d1e098173 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -279,6 +279,9 @@ def host(url)
Settings.cron_jobs['stuck_ci_builds_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['stuck_ci_builds_worker']['cron'] ||= '0 0 * * *'
Settings.cron_jobs['stuck_ci_builds_worker']['job_class'] = 'StuckCiBuildsWorker'
+Settings.cron_jobs['expire_build_artifacts'] ||= Settingslogic.new({})
+Settings.cron_jobs['expire_build_artifacts']['cron'] ||= '0 0 * * *'
+Settings.cron_jobs['expire_build_artifacts']['job_class'] = 'ExpireBuildArtifacts'
Settings.cron_jobs['repository_check_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['repository_check_worker']['cron'] ||= '20 * * * *'
Settings.cron_jobs['repository_check_worker']['job_class'] = 'RepositoryCheck::BatchWorker'
diff --git a/config/routes.rb b/config/routes.rb
index 95fbe7dd9df9c..3d092d98c8ec5 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -714,6 +714,7 @@
post :cancel
post :retry
post :erase
+ post :keep_artifacts
get :trace
get :raw
end
diff --git a/db/migrate/20160518200441_add_artifacts_expire_date_to_ci_builds.rb b/db/migrate/20160518200441_add_artifacts_expire_date_to_ci_builds.rb
new file mode 100644
index 0000000000000..915167b038d48
--- /dev/null
+++ b/db/migrate/20160518200441_add_artifacts_expire_date_to_ci_builds.rb
@@ -0,0 +1,5 @@
+class AddArtifactsExpireDateToCiBuilds < ActiveRecord::Migration
+ def change
+ add_column :ci_builds, :artifacts_expire_at, :timestamp
+ end
+end
diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb
index 607359769d153..54f5626c7d7dc 100644
--- a/lib/ci/api/builds.rb
+++ b/lib/ci/api/builds.rb
@@ -114,6 +114,7 @@ class Builds < Grape::API
# id (required) - The ID of a build
# token (required) - The build authorization token
# file (required) - Artifacts file
+ # expire_in (optional) - Specify when artifacts should expire (ex. 7d)
# Parameters (accelerated by GitLab Workhorse):
# file.path - path to locally stored body (generated by Workhorse)
# file.name - real filename as send in Content-Disposition
@@ -145,6 +146,7 @@ class Builds < Grape::API
build.artifacts_file = artifacts
build.artifacts_metadata = metadata
+ build.artifacts_expire_at = Time.now + ChronicDuration.parse(params['expire_in'])
if build.save
present(build, with: Entities::BuildDetails)
diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb
index 15d57a46eb010..b1297565ebed1 100644
--- a/lib/ci/gitlab_ci_yaml_processor.rb
+++ b/lib/ci/gitlab_ci_yaml_processor.rb
@@ -9,7 +9,7 @@ class ValidationError < StandardError; end
:allow_failure, :type, :stage, :when, :artifacts, :cache,
:dependencies, :before_script, :after_script, :variables]
ALLOWED_CACHE_KEYS = [:key, :untracked, :paths]
- ALLOWED_ARTIFACTS_KEYS = [:name, :untracked, :paths, :when]
+ ALLOWED_ARTIFACTS_KEYS = [:name, :untracked, :paths, :when, :expire_in]
attr_reader :before_script, :after_script, :image, :services, :path, :cache
--
GitLab