diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 22dc2359189f0bdc402371d4467df579ca21572a..179ad9e35876122f21ef8d88f5047f9451d0e19b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,6 +9,7 @@ stages: - deploy .default_variables: &default_variables + CI_API_V4_URL: https://gitlab.com/api/v4 DEFAULT_IMAGE_WITHOUT_TAG: "${CI_REGISTRY_IMAGE}/asdf-bootstrapped-verify" DEFAULT_BRANCH_IMAGE: "${DEFAULT_IMAGE_WITHOUT_TAG}:${CI_COMMIT_REF_SLUG}" DEFAULT_MAIN_IMAGE: "${DEFAULT_IMAGE_WITHOUT_TAG}:main" @@ -33,6 +34,7 @@ stages: GITLAB_LAST_VERIFIED_SHA_PATH: gitlab-last-verified-sha.json REGISTRY_HOST: "registry.gitlab.com" REGISTRY_GROUP: "gitlab-org" + RUBY_VERSION: "3.2" FF_TIMESTAMPS: true variables: diff --git a/.gitlab/ci/_rules.gitlab-ci.yml b/.gitlab/ci/_rules.gitlab-ci.yml index ceb58c97e7c1fdf10bdd8fb4f1b6bbcbbdfafe29..4ce64cca1653f69a5e6b8218745c179a850f79de 100644 --- a/.gitlab/ci/_rules.gitlab-ci.yml +++ b/.gitlab/ci/_rules.gitlab-ci.yml @@ -10,6 +10,9 @@ .if-default-branch-schedule: &if-default-branch-schedule if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "schedule"' +.if-default-branch-schedule-maintenance: &if-default-branch-schedule-maintenance + if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "maintenance"' + .if-release-image: &if-release-image if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "gitpod-image-release"' @@ -123,6 +126,14 @@ - <<: *if-default-branch-schedule - changes: *code-changes +.rules:packages-cleanup: + rules: + - <<: *if-merge-request + when: never + - <<: *if-fork + when: never + - <<: *if-default-branch-schedule-maintenance + .rules:docs-code-changes: rules: - changes: *docs-code-changes diff --git a/.gitlab/ci/compile.gitlab-ci.yml b/.gitlab/ci/compile.gitlab-ci.yml index 8fc5888a84618a260ba3df826fd7fd308c0c4764..a3e9b0ebf70152a2d5c4b2705c8d84d123a589ad 100755 --- a/.gitlab/ci/compile.gitlab-ci.yml +++ b/.gitlab/ci/compile.gitlab-ci.yml @@ -6,7 +6,7 @@ tags: - gitlab-org variables: - REPOSITORY_FILES_API_URL: "https://gitlab.com/api/v4/projects/278964/repository/files" # Refers to https://gitlab.com/gitlab-org/gitlab + REPOSITORY_FILES_API_URL: "${CI_API_V4_URL}/projects/278964/repository/files" # Refers to https://gitlab.com/gitlab-org/gitlab before_script: - apt-get update && apt-get install -y ruby3.1 - gem install gitlab-sdk sentry-ruby zeitwerk tty-spinner @@ -87,3 +87,14 @@ compile:gitaly: paths: - gitaly/checksums.txt expire_in: 14d + +packages-cleanup: + extends: + - .rules:packages-cleanup + stage: pre + image: ruby:${RUBY_VERSION} + before_script: + - gem install httparty --no-document --version 0.20.0 + - gem install gitlab --no-document --version 4.19.0 + script: + - support/package-cleanup diff --git a/support/package-cleanup b/support/package-cleanup new file mode 100755 index 0000000000000000000000000000000000000000..668c2a6461360ac2c3bc283123f90cbb94b1bd35 --- /dev/null +++ b/support/package-cleanup @@ -0,0 +1,129 @@ +#!/usr/bin/env ruby +# +# frozen_string_literal: true + +# Borrowed from https://gitlab.com/gitlab-org/gitlab/-/blob/14ba134c5db017cd52e1eab20584d0db42dd0c02/scripts/packages/automated_cleanup.rb + +require 'gitlab' +require 'optparse' + +module Packages + class AutomatedCleanup + PACKAGES_PER_PAGE = 100 + + def initialize( + project_path: ENV.fetch('CI_PROJECT_PATH', nil), + gitlab_token: ENV.fetch('GDK_PROJECT_PACKAGES_CLEANUP_API_TOKEN', nil), + api_endpoint: ENV.fetch('CI_API_V4_URL', nil), + options: {} + ) + @project_path = project_path + @gitlab_token = gitlab_token + @api_endpoint = api_endpoint + @dry_run = options[:dry_run] + + puts "Dry-run mode." if dry_run + end + + def gitlab + @gitlab ||= begin + Gitlab.configure do |config| + config.endpoint = api_endpoint + config.private_token = gitlab_token + end + + Gitlab + end + end + + def perform_gitlab_package_cleanup!(package_name:, days_for_delete:) + puts "Checking for '#{package_name}' packages created at least #{days_for_delete} days ago..." + + gitlab.project_packages(project_path, + package_type: 'generic', + package_name: package_name, + per_page: PACKAGES_PER_PAGE).auto_paginate do |package| + next unless package.name == package_name # the search is fuzzy, so we better check the actual package name + + delete_package(package) if old_enough(package, days_for_delete) && not_recently_downloaded(package, days_for_delete) + end + end + + private + + attr_reader :project_path, :gitlab_token, :api_endpoint, :dry_run + + def delete_package(package) + print_package_state(package) + gitlab.delete_project_package(project_path, package.id) unless dry_run + rescue Gitlab::Error::Forbidden + puts "Package #{package_full_name(package)} is forbidden: skipping it" + end + + def time_ago(days:) + Time.now - (days * 24 * 3600) + end + + def old_enough(package, days_for_delete) + Time.parse(package.created_at) < time_ago(days: days_for_delete) + end + + def not_recently_downloaded(package, days_for_delete) + package.last_downloaded_at.nil? || + Time.parse(package.last_downloaded_at) < time_ago(days: days_for_delete) + end + + def print_package_state(package) + download_text = + if package.last_downloaded_at + "last downloaded on #{package.last_downloaded_at}" + else + "never downloaded" + end + + puts "\nPackage #{package_full_name(package)} (created on #{package.created_at}) was " \ + "#{download_text}: deleting it.\n" + end + + def package_full_name(package) + "'#{package.name}/#{package.version}'" + end + end +end + +def timed(task) + start = Time.now + yield(self) + puts "#{task} finished in #{Time.now - start} seconds.\n" +end + +if $PROGRAM_NAME == __FILE__ + options = { + dry_run: false + } + + OptionParser.new do |opts| + opts.on("-d", "--dry-run", "Whether to perform a dry-run or not.") do |_value| + options[:dry_run] = true + end + + opts.on("-h", "--help", "Prints this help") do + puts opts + exit + end + end.parse! + + automated_cleanup = Packages::AutomatedCleanup.new(options: options) + + timed('"gitaly" packages cleanup') do + automated_cleanup.perform_gitlab_package_cleanup!(package_name: 'gitaly', days_for_delete: 30) + end + + timed('"gitlab-shell" packages cleanup') do + automated_cleanup.perform_gitlab_package_cleanup!(package_name: 'gitlab-shell', days_for_delete: 30) + end + + timed('"workhorse" packages cleanup') do + automated_cleanup.perform_gitlab_package_cleanup!(package_name: 'workhorse', days_for_delete: 30) + end +end