From 1bb608daaafa804a28e6d72b01284c0b52c9eb4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= <remy@rymai.me> Date: Tue, 4 Oct 2022 18:01:03 +0100 Subject: [PATCH] ci: Add 'scripts/packages/automated_cleanup.rb' to cleanup packages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable <remy@rymai.me> --- scripts/packages/automated_cleanup.rb | 120 +++++++ scripts/review_apps/automated_cleanup.rb | 416 ++++++++++++----------- scripts/utils.sh | 6 +- 3 files changed, 343 insertions(+), 199 deletions(-) create mode 100755 scripts/packages/automated_cleanup.rb diff --git a/scripts/packages/automated_cleanup.rb b/scripts/packages/automated_cleanup.rb new file mode 100755 index 0000000000000..ce8a23315a916 --- /dev/null +++ b/scripts/packages/automated_cleanup.rb @@ -0,0 +1,120 @@ +#!/usr/bin/env ruby + +# frozen_string_literal: true + +require 'optparse' +require 'gitlab' + +module Packages + class AutomatedCleanup + PACKAGES_PER_PAGE = 100 + + # $GITLAB_PROJECT_PACKAGES_CLEANUP_API_TOKEN => `Packages Cleanup` project token + def initialize( + project_path: ENV['CI_PROJECT_PATH'], + gitlab_token: ENV['GITLAB_PROJECT_PACKAGES_CLEANUP_API_TOKEN'], + options: {} + ) + @project_path = project_path + @gitlab_token = gitlab_token + @dry_run = options[:dry_run] + + puts "Dry-run mode." if dry_run + end + + def gitlab + @gitlab ||= begin + Gitlab.configure do |config| + config.endpoint = 'https://gitlab.com/api/v4' + config.private_token = gitlab_token + end + + Gitlab + end + end + + def perform_gitlab_package_cleanup!(package_name:, days_for_delete:) + puts "Checking for '#{package_name}' packages created at least #{days_for_delete} days ago..." + + gitlab.project_packages(project_path, + package_type: 'generic', + package_name: package_name, + per_page: PACKAGES_PER_PAGE).auto_paginate do |package| + next unless package.name == package_name # the search is fuzzy, so we better check the actual package name + + if old_enough(package, days_for_delete) && not_recently_downloaded(package, days_for_delete) + delete_package(package) + end + end + end + + private + + attr_reader :project_path, :gitlab_token, :dry_run + + def delete_package(package) + print_package_state(package) + gitlab.delete_project_package(project_path, package.id) unless dry_run + rescue Gitlab::Error::Forbidden + puts "Package #{package_full_name(package)} is forbidden: skipping it" + end + + def time_ago(days:) + Time.now - days * 24 * 3600 + end + + def old_enough(package, days_for_delete) + Time.parse(package.created_at) < time_ago(days: days_for_delete) + end + + def not_recently_downloaded(package, days_for_delete) + package.last_downloaded_at.nil? || + Time.parse(package.last_downloaded_at) < time_ago(days: days_for_delete) + end + + def print_package_state(package) + download_text = + if package.last_downloaded_at + "last downloaded on #{package.last_downloaded_at}" + else + "never downloaded" + end + + puts "\nPackage #{package_full_name(package)} (created on #{package.created_at}) was " \ + "#{download_text}: deleting it.\n" + end + + def package_full_name(package) + "'#{package.name}/#{package.version}'" + end + end +end + +def timed(task) + start = Time.now + yield(self) + puts "#{task} finished in #{Time.now - start} seconds.\n" +end + +if $0 == __FILE__ + options = { + dry_run: false + } + + OptionParser.new do |opts| + opts.on("-d", "--dry-run", "Whether to perform a dry-run or not.") do |value| + options[:dry_run] = true + end + + opts.on("-h", "--help", "Prints this help") do + puts opts + exit + end + end.parse! + + automated_cleanup = Packages::AutomatedCleanup.new(options: options) + + timed('"assets" packages cleanup') do + automated_cleanup.perform_gitlab_package_cleanup!(package_name: 'assets', days_for_delete: 7) + end +end diff --git a/scripts/review_apps/automated_cleanup.rb b/scripts/review_apps/automated_cleanup.rb index e6efbca9e8647..d8b46556b4a7b 100755 --- a/scripts/review_apps/automated_cleanup.rb +++ b/scripts/review_apps/automated_cleanup.rb @@ -1,252 +1,261 @@ +#!/usr/bin/env ruby + # frozen_string_literal: true +require 'optparse' require 'gitlab' require_relative File.expand_path('../../tooling/lib/tooling/helm3_client.rb', __dir__) require_relative File.expand_path('../../tooling/lib/tooling/kubernetes_client.rb', __dir__) -class AutomatedCleanup - attr_reader :project_path, :gitlab_token - - DEPLOYMENTS_PER_PAGE = 100 - ENVIRONMENT_PREFIX = { - review_app: 'review/', - docs_review_app: 'review-docs/' - }.freeze - IGNORED_HELM_ERRORS = [ - 'transport is closing', - 'error upgrading connection', - 'not found' - ].freeze - IGNORED_KUBERNETES_ERRORS = [ - 'NotFound' - ].freeze - - def self.ee? - # Support former project name for `dev` - %w[gitlab gitlab-ee].include?(ENV['CI_PROJECT_NAME']) - end +module ReviewApps + class AutomatedCleanup + DEPLOYMENTS_PER_PAGE = 100 + ENVIRONMENT_PREFIX = { + review_app: 'review/', + docs_review_app: 'review-docs/' + }.freeze + IGNORED_HELM_ERRORS = [ + 'transport is closing', + 'error upgrading connection', + 'not found' + ].freeze + IGNORED_KUBERNETES_ERRORS = [ + 'NotFound' + ].freeze + + # $GITLAB_PROJECT_REVIEW_APP_CLEANUP_API_TOKEN => `Automated Review App Cleanup` project token + def initialize( + project_path: ENV['CI_PROJECT_PATH'], + gitlab_token: ENV['GITLAB_PROJECT_REVIEW_APP_CLEANUP_API_TOKEN'], + options: {} + ) + @project_path = project_path + @gitlab_token = gitlab_token + @dry_run = options[:dry_run] + + puts "Dry-run mode." if dry_run + end - # $GITLAB_PROJECT_REVIEW_APP_CLEANUP_API_TOKEN => `Automated Review App Cleanup` project token - def initialize(project_path: ENV['CI_PROJECT_PATH'], gitlab_token: ENV['GITLAB_PROJECT_REVIEW_APP_CLEANUP_API_TOKEN']) - @project_path = project_path - @gitlab_token = gitlab_token - end + def gitlab + @gitlab ||= begin + Gitlab.configure do |config| + config.endpoint = 'https://gitlab.com/api/v4' + # gitlab-bot's token "GitLab review apps cleanup" + config.private_token = gitlab_token + end - def gitlab - @gitlab ||= begin - Gitlab.configure do |config| - config.endpoint = 'https://gitlab.com/api/v4' - # gitlab-bot's token "GitLab review apps cleanup" - config.private_token = gitlab_token + Gitlab end - - Gitlab end - end - def review_apps_namespace - 'review-apps' - end + def review_apps_namespace + 'review-apps' + end - def helm - @helm ||= Tooling::Helm3Client.new(namespace: review_apps_namespace) - end + def helm + @helm ||= Tooling::Helm3Client.new(namespace: review_apps_namespace) + end - def kubernetes - @kubernetes ||= Tooling::KubernetesClient.new(namespace: review_apps_namespace) - end + def kubernetes + @kubernetes ||= Tooling::KubernetesClient.new(namespace: review_apps_namespace) + end - def perform_gitlab_environment_cleanup!(days_for_stop:, days_for_delete:) - puts "Checking for Review Apps not updated in the last #{days_for_stop} days..." + def perform_gitlab_environment_cleanup!(days_for_stop:, days_for_delete:) + puts "Checking for Review Apps not updated in the last #{days_for_stop} days..." - checked_environments = [] - delete_threshold = threshold_time(days: days_for_delete) - stop_threshold = threshold_time(days: days_for_stop) - deployments_look_back_threshold = threshold_time(days: days_for_delete * 5) + checked_environments = [] + delete_threshold = threshold_time(days: days_for_delete) + stop_threshold = threshold_time(days: days_for_stop) + deployments_look_back_threshold = threshold_time(days: days_for_delete * 5) - releases_to_delete = [] + releases_to_delete = [] - # Delete environments via deployments - gitlab.deployments(project_path, per_page: DEPLOYMENTS_PER_PAGE, sort: 'desc').auto_paginate do |deployment| - break if Time.parse(deployment.created_at) < deployments_look_back_threshold + # Delete environments via deployments + gitlab.deployments(project_path, per_page: DEPLOYMENTS_PER_PAGE, sort: 'desc').auto_paginate do |deployment| + break if Time.parse(deployment.created_at) < deployments_look_back_threshold - environment = deployment.environment + environment = deployment.environment - next unless environment - next unless environment.name.start_with?(ENVIRONMENT_PREFIX[:review_app]) - next if checked_environments.include?(environment.slug) + next unless environment + next unless environment.name.start_with?(ENVIRONMENT_PREFIX[:review_app]) + next if checked_environments.include?(environment.slug) - last_deploy = deployment.created_at - deployed_at = Time.parse(last_deploy) + last_deploy = deployment.created_at + deployed_at = Time.parse(last_deploy) - if deployed_at < delete_threshold - deleted_environment = delete_environment(environment, deployment) - if deleted_environment - release = Tooling::Helm3Client::Release.new(environment.slug, 1, deployed_at.to_s, nil, nil, review_apps_namespace) - releases_to_delete << release - end - else - if deployed_at >= stop_threshold - print_release_state(subject: 'Review App', release_name: environment.slug, release_date: last_deploy, action: 'leaving') + if deployed_at < delete_threshold + deleted_environment = delete_environment(environment, deployment) + if deleted_environment + release = Tooling::Helm3Client::Release.new(environment.slug, 1, deployed_at.to_s, nil, nil, review_apps_namespace) + releases_to_delete << release + end else - environment_state = fetch_environment(environment)&.state - stop_environment(environment, deployment) if environment_state && environment_state != 'stopped' + if deployed_at >= stop_threshold + print_release_state(subject: 'Review App', release_name: environment.slug, release_date: last_deploy, action: 'leaving') + else + environment_state = fetch_environment(environment)&.state + stop_environment(environment, deployment) if environment_state && environment_state != 'stopped' + end end + + checked_environments << environment.slug end - checked_environments << environment.slug - end + delete_stopped_environments(environment_type: :review_app, checked_environments: checked_environments, last_updated_threshold: delete_threshold) do |environment| + releases_to_delete << Tooling::Helm3Client::Release.new(environment.slug, 1, environment.updated_at, nil, nil, review_apps_namespace) + end - delete_stopped_environments(environment_type: :review_app, checked_environments: checked_environments, last_updated_threshold: delete_threshold) do |environment| - releases_to_delete << Tooling::Helm3Client::Release.new(environment.slug, 1, environment.updated_at, nil, nil, review_apps_namespace) + delete_helm_releases(releases_to_delete) end - delete_helm_releases(releases_to_delete) - end + def perform_gitlab_docs_environment_cleanup!(days_for_stop:, days_for_delete:) + puts "Checking for Docs Review Apps not updated in the last #{days_for_stop} days..." - def perform_gitlab_docs_environment_cleanup!(days_for_stop:, days_for_delete:) - puts "Checking for Docs Review Apps not updated in the last #{days_for_stop} days..." + checked_environments = [] + stop_threshold = threshold_time(days: days_for_stop) + delete_threshold = threshold_time(days: days_for_delete) - checked_environments = [] - stop_threshold = threshold_time(days: days_for_stop) - delete_threshold = threshold_time(days: days_for_delete) + # Delete environments via deployments + gitlab.deployments(project_path, per_page: DEPLOYMENTS_PER_PAGE, sort: 'desc').auto_paginate do |deployment| + environment = deployment.environment - # Delete environments via deployments - gitlab.deployments(project_path, per_page: DEPLOYMENTS_PER_PAGE, sort: 'desc').auto_paginate do |deployment| - environment = deployment.environment + next unless environment + next unless environment.name.start_with?(ENVIRONMENT_PREFIX[:docs_review_app]) + next if checked_environments.include?(environment.slug) - next unless environment - next unless environment.name.start_with?(ENVIRONMENT_PREFIX[:docs_review_app]) - next if checked_environments.include?(environment.slug) + last_deploy = deployment.created_at + deployed_at = Time.parse(last_deploy) - last_deploy = deployment.created_at - deployed_at = Time.parse(last_deploy) + if deployed_at < stop_threshold + environment_state = fetch_environment(environment)&.state + stop_environment(environment, deployment) if environment_state && environment_state != 'stopped' + end - if deployed_at < stop_threshold - environment_state = fetch_environment(environment)&.state - stop_environment(environment, deployment) if environment_state && environment_state != 'stopped' - end + delete_environment(environment, deployment) if deployed_at < delete_threshold - delete_environment(environment, deployment) if deployed_at < delete_threshold + checked_environments << environment.slug + end - checked_environments << environment.slug + delete_stopped_environments(environment_type: :docs_review_app, checked_environments: checked_environments, last_updated_threshold: delete_threshold) end - delete_stopped_environments(environment_type: :docs_review_app, checked_environments: checked_environments, last_updated_threshold: delete_threshold) - end - - def perform_helm_releases_cleanup!(days:) - puts "Checking for Helm releases that are failed or not updated in the last #{days} days..." + def perform_helm_releases_cleanup!(days:) + puts "Checking for Helm releases that are failed or not updated in the last #{days} days..." - threshold = threshold_time(days: days) + threshold = threshold_time(days: days) - releases_to_delete = [] + releases_to_delete = [] - helm_releases.each do |release| - # Prevents deleting `dns-gitlab-review-app` releases or other unrelated releases - next unless release.name.start_with?('review-') + helm_releases.each do |release| + # Prevents deleting `dns-gitlab-review-app` releases or other unrelated releases + next unless release.name.start_with?('review-') - if release.status == 'failed' || release.last_update < threshold - releases_to_delete << release - else - print_release_state(subject: 'Release', release_name: release.name, release_date: release.last_update, action: 'leaving') + if release.status == 'failed' || release.last_update < threshold + releases_to_delete << release + else + print_release_state(subject: 'Release', release_name: release.name, release_date: release.last_update, action: 'leaving') + end end + + delete_helm_releases(releases_to_delete) end - delete_helm_releases(releases_to_delete) - end + def perform_stale_namespace_cleanup!(days:) + kubernetes_client = Tooling::KubernetesClient.new(namespace: nil) - def perform_stale_namespace_cleanup!(days:) - kubernetes_client = Tooling::KubernetesClient.new(namespace: nil) + kubernetes_client.cleanup_review_app_namespaces(created_before: threshold_time(days: days), wait: false) unless dry_run + end - kubernetes_client.cleanup_review_app_namespaces(created_before: threshold_time(days: days), wait: false) - end + def perform_stale_pvc_cleanup!(days:) + kubernetes.cleanup_by_created_at(resource_type: 'pvc', created_before: threshold_time(days: days), wait: false) unless dry_run + end - def perform_stale_pvc_cleanup!(days:) - kubernetes.cleanup_by_created_at(resource_type: 'pvc', created_before: threshold_time(days: days), wait: false) - end + private - private + attr_reader :project_path, :gitlab_token, :dry_run - def fetch_environment(environment) - gitlab.environment(project_path, environment.id) - rescue Errno::ETIMEDOUT => ex - puts "Failed to fetch '#{environment.name}' / '#{environment.slug}' (##{environment.id}):\n#{ex.message}" - nil - end + def fetch_environment(environment) + gitlab.environment(project_path, environment.id) + rescue Errno::ETIMEDOUT => ex + puts "Failed to fetch '#{environment.name}' / '#{environment.slug}' (##{environment.id}):\n#{ex.message}" + nil + end - def delete_environment(environment, deployment = nil) - release_date = deployment ? deployment.created_at : environment.updated_at - print_release_state(subject: 'Review app', release_name: environment.slug, release_date: release_date, action: 'deleting') - gitlab.delete_environment(project_path, environment.id) + def delete_environment(environment, deployment = nil) + release_date = deployment ? deployment.created_at : environment.updated_at + print_release_state(subject: 'Review app', release_name: environment.slug, release_date: release_date, action: 'deleting') + gitlab.delete_environment(project_path, environment.id) unless dry_run - rescue Gitlab::Error::Forbidden - puts "Review app '#{environment.name}' / '#{environment.slug}' (##{environment.id}) is forbidden: skipping it" - end + rescue Gitlab::Error::Forbidden + puts "Review app '#{environment.name}' / '#{environment.slug}' (##{environment.id}) is forbidden: skipping it" + end - def stop_environment(environment, deployment) - print_release_state(subject: 'Review app', release_name: environment.slug, release_date: deployment.created_at, action: 'stopping') - gitlab.stop_environment(project_path, environment.id) + def stop_environment(environment, deployment) + print_release_state(subject: 'Review app', release_name: environment.slug, release_date: deployment.created_at, action: 'stopping') + gitlab.stop_environment(project_path, environment.id) unless dry_run - rescue Gitlab::Error::Forbidden - puts "Review app '#{environment.name}' / '#{environment.slug}' (##{environment.id}) is forbidden: skipping it" - end + rescue Gitlab::Error::Forbidden + puts "Review app '#{environment.name}' / '#{environment.slug}' (##{environment.id}) is forbidden: skipping it" + end - def delete_stopped_environments(environment_type:, checked_environments:, last_updated_threshold:) - gitlab.environments(project_path, per_page: DEPLOYMENTS_PER_PAGE, sort: 'desc', states: 'stopped', search: ENVIRONMENT_PREFIX[environment_type]).auto_paginate do |environment| - next if skip_environment?(environment: environment, checked_environments: checked_environments, last_updated_threshold: last_updated_threshold, environment_type: environment_type) + def delete_stopped_environments(environment_type:, checked_environments:, last_updated_threshold:) + gitlab.environments(project_path, per_page: DEPLOYMENTS_PER_PAGE, sort: 'desc', states: 'stopped', search: ENVIRONMENT_PREFIX[environment_type]).auto_paginate do |environment| + next if skip_environment?(environment: environment, checked_environments: checked_environments, last_updated_threshold: last_updated_threshold, environment_type: environment_type) - yield environment if delete_environment(environment) + yield environment if delete_environment(environment) - checked_environments << environment.slug + checked_environments << environment.slug + end end - end - def skip_environment?(environment:, checked_environments:, last_updated_threshold:, environment_type:) - return true unless environment.name.start_with?(ENVIRONMENT_PREFIX[environment_type]) - return true if checked_environments.include?(environment.slug) - return true if Time.parse(environment.updated_at) > last_updated_threshold + def skip_environment?(environment:, checked_environments:, last_updated_threshold:, environment_type:) + return true unless environment.name.start_with?(ENVIRONMENT_PREFIX[environment_type]) + return true if checked_environments.include?(environment.slug) + return true if Time.parse(environment.updated_at) > last_updated_threshold - false - end + false + end - def helm_releases - args = ['--all', '--date'] + def helm_releases + args = ['--all', '--date'] - helm.releases(args: args) - end + helm.releases(args: args) + end - def delete_helm_releases(releases) - return if releases.empty? + def delete_helm_releases(releases) + return if releases.empty? - releases.each do |release| - print_release_state(subject: 'Release', release_name: release.name, release_status: release.status, release_date: release.last_update, action: 'cleaning') - end + releases.each do |release| + print_release_state(subject: 'Release', release_name: release.name, release_status: release.status, release_date: release.last_update, action: 'cleaning') + end - releases_names = releases.map(&:name) - helm.delete(release_name: releases_names) - kubernetes.cleanup_by_release(release_name: releases_names, wait: false) + releases_names = releases.map(&:name) + unless dry_run + helm.delete(release_name: releases_names) + kubernetes.cleanup_by_release(release_name: releases_names, wait: false) + end - rescue Tooling::Helm3Client::CommandFailedError => ex - raise ex unless ignore_exception?(ex.message, IGNORED_HELM_ERRORS) + rescue Tooling::Helm3Client::CommandFailedError => ex + raise ex unless ignore_exception?(ex.message, IGNORED_HELM_ERRORS) - puts "Ignoring the following Helm error:\n#{ex}\n" - rescue Tooling::KubernetesClient::CommandFailedError => ex - raise ex unless ignore_exception?(ex.message, IGNORED_KUBERNETES_ERRORS) + puts "Ignoring the following Helm error:\n#{ex}\n" + rescue Tooling::KubernetesClient::CommandFailedError => ex + raise ex unless ignore_exception?(ex.message, IGNORED_KUBERNETES_ERRORS) - puts "Ignoring the following Kubernetes error:\n#{ex}\n" - end + puts "Ignoring the following Kubernetes error:\n#{ex}\n" + end - def threshold_time(days:) - Time.now - days * 24 * 3600 - end + def threshold_time(days:) + Time.now - days * 24 * 3600 + end - def ignore_exception?(exception_message, exceptions_ignored) - exception_message.match?(/(#{exceptions_ignored})/) - end + def ignore_exception?(exception_message, exceptions_ignored) + exception_message.match?(/(#{exceptions_ignored})/) + end - def print_release_state(subject:, release_name:, release_date:, action:, release_status: nil) - puts "\n#{subject} '#{release_name}' #{"(#{release_status}) " if release_status}was last deployed on #{release_date}: #{action} it.\n" + def print_release_state(subject:, release_name:, release_date:, action:, release_status: nil) + puts "\n#{subject} '#{release_name}' #{"(#{release_status}) " if release_status}was last deployed on #{release_date}: #{action} it.\n" + end end end @@ -256,28 +265,43 @@ def timed(task) puts "#{task} finished in #{Time.now - start} seconds.\n" end -automated_cleanup = AutomatedCleanup.new +if $0 == __FILE__ + options = { + dry_run: false + } -timed('Review Apps cleanup') do - automated_cleanup.perform_gitlab_environment_cleanup!(days_for_stop: 5, days_for_delete: 6) -end + OptionParser.new do |opts| + opts.on("-d", "--dry-run", "Whether to perform a dry-run or not.") do |value| + options[:dry_run] = true + end -timed('Docs Review Apps cleanup') do - automated_cleanup.perform_gitlab_docs_environment_cleanup!(days_for_stop: 20, days_for_delete: 30) -end + opts.on("-h", "--help", "Prints this help") do + puts opts + exit + end + end.parse! -puts + automated_cleanup = ReviewApps::AutomatedCleanup.new(options: options) -timed('Helm releases cleanup') do - automated_cleanup.perform_helm_releases_cleanup!(days: 7) -end + timed('Review Apps cleanup') do + automated_cleanup.perform_gitlab_environment_cleanup!(days_for_stop: 5, days_for_delete: 6) + end -timed('Stale Namespace cleanup') do - automated_cleanup.perform_stale_namespace_cleanup!(days: 14) -end + timed('Docs Review Apps cleanup') do + automated_cleanup.perform_gitlab_docs_environment_cleanup!(days_for_stop: 20, days_for_delete: 30) + end -timed('Stale PVC cleanup') do - automated_cleanup.perform_stale_pvc_cleanup!(days: 30) -end + puts + + timed('Helm releases cleanup') do + automated_cleanup.perform_helm_releases_cleanup!(days: 7) + end -exit(0) + timed('Stale Namespace cleanup') do + automated_cleanup.perform_stale_namespace_cleanup!(days: 14) + end + + timed('Stale PVC cleanup') do + automated_cleanup.perform_stale_pvc_cleanup!(days: 30) + end +end diff --git a/scripts/utils.sh b/scripts/utils.sh index 10b7f856ee64d..ff1e28b7a9393 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -77,12 +77,12 @@ function setup_db() { } function install_gitlab_gem() { - run_timed_command "gem install httparty --no-document --version 0.18.1" - run_timed_command "gem install gitlab --no-document --version 4.17.0" + run_timed_command "gem install httparty --no-document --version 0.20.0" + run_timed_command "gem install gitlab --no-document --version 4.19.0" } function install_tff_gem() { - run_timed_command "gem install test_file_finder --no-document --version 0.1.1" + run_timed_command "gem install test_file_finder --no-document --version 0.1.4" } function install_junit_merge_gem() { -- GitLab