From 951c7bfd13352e5dd74ddfa5eb3eaa3effffe284 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= <matteeyah@gmail.com>
Date: Tue, 13 Apr 2021 18:02:13 +0000
Subject: [PATCH] Schedule artifact expiry backfill again
The previous background migration left around 400.000 artifacts without
an expiry date. This schedules the background migration again to
backfill those entries as well.
---
...e-reschedule-artifac-expiry-date-again.yml | 5 +++
...schedule_artifact_expiry_backfill_again.rb | 44 +++++++++++++++++++
db/schema_migrations/20210413132500 | 1 +
...ule_artifact_expiry_backfill_again_spec.rb | 38 ++++++++++++++++
4 files changed, 88 insertions(+)
create mode 100644 changelogs/unreleased/mc-backstage-reschedule-artifac-expiry-date-again.yml
create mode 100644 db/post_migrate/20210413132500_reschedule_artifact_expiry_backfill_again.rb
create mode 100644 db/schema_migrations/20210413132500
create mode 100644 spec/migrations/20210413132500_reschedule_artifact_expiry_backfill_again_spec.rb
diff --git a/changelogs/unreleased/mc-backstage-reschedule-artifac-expiry-date-again.yml b/changelogs/unreleased/mc-backstage-reschedule-artifac-expiry-date-again.yml
new file mode 100644
index 0000000000000..ffdd268c086bc
--- /dev/null
+++ b/changelogs/unreleased/mc-backstage-reschedule-artifac-expiry-date-again.yml
@@ -0,0 +1,5 @@
+---
+title: Schedule artifact expiry backfill again.
+merge_request: 59270
+author:
+type: changed
diff --git a/db/post_migrate/20210413132500_reschedule_artifact_expiry_backfill_again.rb b/db/post_migrate/20210413132500_reschedule_artifact_expiry_backfill_again.rb
new file mode 100644
index 0000000000000..b4570c8398b8e
--- /dev/null
+++ b/db/post_migrate/20210413132500_reschedule_artifact_expiry_backfill_again.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+class RescheduleArtifactExpiryBackfillAgain < ActiveRecord::Migration[6.0]
+ include Gitlab::Database::MigrationHelpers
+
+ DOWNTIME = false
+ MIGRATION = 'BackfillArtifactExpiryDate'
+ SWITCH_DATE = Date.new(2020, 06, 22).freeze
+
+ disable_ddl_transaction!
+
+ class JobArtifact < ActiveRecord::Base
+ include EachBatch
+
+ self.inheritance_column = :_type_disabled
+ self.table_name = 'ci_job_artifacts'
+
+ scope :without_expiry_date, -> { where(expire_at: nil) }
+ scope :before_switch, -> { where("date(created_at AT TIME ZONE 'UTC') < ?::date", SWITCH_DATE) }
+ end
+
+ def up
+ Gitlab::BackgroundMigration.steal(MIGRATION) do |job|
+ job.delete
+
+ false
+ end
+
+ queue_background_migration_jobs_by_range_at_intervals(
+ JobArtifact.without_expiry_date.before_switch,
+ MIGRATION,
+ 2.minutes,
+ batch_size: 200_000
+ )
+ end
+
+ def down
+ Gitlab::BackgroundMigration.steal(MIGRATION) do |job|
+ job.delete
+
+ false
+ end
+ end
+end
diff --git a/db/schema_migrations/20210413132500 b/db/schema_migrations/20210413132500
new file mode 100644
index 0000000000000..662c7e33ef0bd
--- /dev/null
+++ b/db/schema_migrations/20210413132500
@@ -0,0 +1 @@
+407806cc168ef9859c9a4f1bd4db7a56aee01367e784ea0767889863b9ace35d
\ No newline at end of file
diff --git a/spec/migrations/20210413132500_reschedule_artifact_expiry_backfill_again_spec.rb b/spec/migrations/20210413132500_reschedule_artifact_expiry_backfill_again_spec.rb
new file mode 100644
index 0000000000000..4f36a95f9cf77
--- /dev/null
+++ b/spec/migrations/20210413132500_reschedule_artifact_expiry_backfill_again_spec.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+require Rails.root.join('db', 'post_migrate', '20210413132500_reschedule_artifact_expiry_backfill_again.rb')
+
+RSpec.describe RescheduleArtifactExpiryBackfillAgain, :migration do
+ let(:migration_class) { Gitlab::BackgroundMigration::BackfillArtifactExpiryDate }
+ let(:migration_name) { migration_class.to_s.demodulize }
+
+ before do
+ table(:namespaces).create!(id: 123, name: 'test_namespace', path: 'test_namespace')
+ table(:projects).create!(id: 123, name: 'sample_project', path: 'sample_project', namespace_id: 123)
+ end
+
+ it 'correctly schedules background migrations' do
+ first_artifact = create_artifact(job_id: 0, expire_at: nil, created_at: Date.new(2020, 06, 21))
+ second_artifact = create_artifact(job_id: 1, expire_at: nil, created_at: Date.new(2020, 06, 21))
+ create_artifact(job_id: 2, expire_at: Date.yesterday, created_at: Date.new(2020, 06, 21))
+ create_artifact(job_id: 3, expire_at: nil, created_at: Date.new(2020, 06, 23))
+
+ Sidekiq::Testing.fake! do
+ freeze_time do
+ migrate!
+
+ expect(BackgroundMigrationWorker.jobs.size).to eq(1)
+ expect(migration_name).to be_scheduled_migration_with_multiple_args(first_artifact.id, second_artifact.id)
+ end
+ end
+ end
+
+ private
+
+ def create_artifact(params)
+ table(:ci_builds).create!(id: params[:job_id], project_id: 123)
+ table(:ci_job_artifacts).create!(project_id: 123, file_type: 1, **params)
+ end
+end
--
GitLab