From 8fd9a9ae92affb3949933490d1758430eaf2d29c Mon Sep 17 00:00:00 2001
From: Maxime Orefice <morefice@gitlab.com>
Date: Tue, 30 Mar 2021 22:50:48 +0000
Subject: [PATCH] Add geo table for pipeline artifact replication
This commit adds all the necessary database changes required to
replicate pipeline artifacts with geo.
---
.../mo-add-pipeline-artifact-geo-database.yml | 5 +++
...ification_state_to_ci_pipeline_artifact.rb | 18 ++++++++
...n_failure_limit_to_ci_pipeline_artifact.rb | 19 +++++++++
...cation_indexes_to_ci_pipeline_artifacts.rb | 27 ++++++++++++
db/schema_migrations/20210325150837 | 1 +
db/schema_migrations/20210325151758 | 1 +
db/schema_migrations/20210325152011 | 1 +
db/structure.sql | 18 +++++++-
...50435_create_pipeline_artifact_registry.rb | 41 +++++++++++++++++++
ee/db/geo/schema.rb | 4 +-
10 files changed, 132 insertions(+), 3 deletions(-)
create mode 100644 changelogs/unreleased/mo-add-pipeline-artifact-geo-database.yml
create mode 100644 db/migrate/20210325150837_add_verification_state_to_ci_pipeline_artifact.rb
create mode 100644 db/migrate/20210325151758_add_verification_failure_limit_to_ci_pipeline_artifact.rb
create mode 100644 db/migrate/20210325152011_add_verification_indexes_to_ci_pipeline_artifacts.rb
create mode 100644 db/schema_migrations/20210325150837
create mode 100644 db/schema_migrations/20210325151758
create mode 100644 db/schema_migrations/20210325152011
create mode 100644 ee/db/geo/migrate/20210325150435_create_pipeline_artifact_registry.rb
diff --git a/changelogs/unreleased/mo-add-pipeline-artifact-geo-database.yml b/changelogs/unreleased/mo-add-pipeline-artifact-geo-database.yml
new file mode 100644
index 0000000000000..2fd629aa3ad0d
--- /dev/null
+++ b/changelogs/unreleased/mo-add-pipeline-artifact-geo-database.yml
@@ -0,0 +1,5 @@
+---
+title: Add geo database changes for pipeline artifact replication
+merge_request: 57506
+author:
+type: added
diff --git a/db/migrate/20210325150837_add_verification_state_to_ci_pipeline_artifact.rb b/db/migrate/20210325150837_add_verification_state_to_ci_pipeline_artifact.rb
new file mode 100644
index 0000000000000..f6c506e59b789
--- /dev/null
+++ b/db/migrate/20210325150837_add_verification_state_to_ci_pipeline_artifact.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class AddVerificationStateToCiPipelineArtifact < ActiveRecord::Migration[6.0]
+ DOWNTIME = false
+
+ def change
+ change_table(:ci_pipeline_artifacts, bulk: true) do |t|
+ t.column :verification_started_at, :datetime_with_timezone
+ t.column :verification_retry_at, :datetime_with_timezone
+ t.column :verified_at, :datetime_with_timezone
+ t.integer :verification_state, default: 0, limit: 2, null: false
+ t.integer :verification_retry_count, limit: 2
+ t.binary :verification_checksum, using: 'verification_checksum::bytea'
+
+ t.text :verification_failure # rubocop:disable Migration/AddLimitToTextColumns
+ end
+ end
+end
diff --git a/db/migrate/20210325151758_add_verification_failure_limit_to_ci_pipeline_artifact.rb b/db/migrate/20210325151758_add_verification_failure_limit_to_ci_pipeline_artifact.rb
new file mode 100644
index 0000000000000..bc4dee2181faa
--- /dev/null
+++ b/db/migrate/20210325151758_add_verification_failure_limit_to_ci_pipeline_artifact.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class AddVerificationFailureLimitToCiPipelineArtifact < ActiveRecord::Migration[6.0]
+ include Gitlab::Database::MigrationHelpers
+
+ DOWNTIME = false
+
+ disable_ddl_transaction!
+
+ CONSTRAINT_NAME = 'ci_pipeline_artifacts_verification_failure_text_limit'
+
+ def up
+ add_text_limit :ci_pipeline_artifacts, :verification_failure, 255, constraint_name: CONSTRAINT_NAME
+ end
+
+ def down
+ remove_check_constraint(:ci_pipeline_artifacts, CONSTRAINT_NAME)
+ end
+end
diff --git a/db/migrate/20210325152011_add_verification_indexes_to_ci_pipeline_artifacts.rb b/db/migrate/20210325152011_add_verification_indexes_to_ci_pipeline_artifacts.rb
new file mode 100644
index 0000000000000..0822aee35a428
--- /dev/null
+++ b/db/migrate/20210325152011_add_verification_indexes_to_ci_pipeline_artifacts.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class AddVerificationIndexesToCiPipelineArtifacts < ActiveRecord::Migration[6.0]
+ include Gitlab::Database::MigrationHelpers
+
+ DOWNTIME = false
+ VERIFICATION_STATE_INDEX_NAME = "index_ci_pipeline_artifacts_verification_state"
+ PENDING_VERIFICATION_INDEX_NAME = "index_ci_pipeline_artifacts_pending_verification"
+ FAILED_VERIFICATION_INDEX_NAME = "index_ci_pipeline_artifacts_failed_verification"
+ NEEDS_VERIFICATION_INDEX_NAME = "index_ci_pipeline_artifacts_needs_verification"
+
+ disable_ddl_transaction!
+
+ def up
+ add_concurrent_index :ci_pipeline_artifacts, :verification_state, name: VERIFICATION_STATE_INDEX_NAME
+ add_concurrent_index :ci_pipeline_artifacts, :verified_at, where: "(verification_state = 0)", order: { verified_at: 'ASC NULLS FIRST' }, name: PENDING_VERIFICATION_INDEX_NAME
+ add_concurrent_index :ci_pipeline_artifacts, :verification_retry_at, where: "(verification_state = 3)", order: { verification_retry_at: 'ASC NULLS FIRST' }, name: FAILED_VERIFICATION_INDEX_NAME
+ add_concurrent_index :ci_pipeline_artifacts, :verification_state, where: "(verification_state = 0 OR verification_state = 3)", name: NEEDS_VERIFICATION_INDEX_NAME
+ end
+
+ def down
+ remove_concurrent_index_by_name :ci_pipeline_artifacts, VERIFICATION_STATE_INDEX_NAME
+ remove_concurrent_index_by_name :ci_pipeline_artifacts, PENDING_VERIFICATION_INDEX_NAME
+ remove_concurrent_index_by_name :ci_pipeline_artifacts, FAILED_VERIFICATION_INDEX_NAME
+ remove_concurrent_index_by_name :ci_pipeline_artifacts, NEEDS_VERIFICATION_INDEX_NAME
+ end
+end
diff --git a/db/schema_migrations/20210325150837 b/db/schema_migrations/20210325150837
new file mode 100644
index 0000000000000..716462ad18711
--- /dev/null
+++ b/db/schema_migrations/20210325150837
@@ -0,0 +1 @@
+6022464130d7a5697f52b9238837c6a6d3363fd349cbcb14052ff52de6ea2e59
\ No newline at end of file
diff --git a/db/schema_migrations/20210325151758 b/db/schema_migrations/20210325151758
new file mode 100644
index 0000000000000..a3d5f1b1b0a7f
--- /dev/null
+++ b/db/schema_migrations/20210325151758
@@ -0,0 +1 @@
+de55a114773961e6cae9ebae36ac93e60676555fe4c2973527511bb3a2eae69d
\ No newline at end of file
diff --git a/db/schema_migrations/20210325152011 b/db/schema_migrations/20210325152011
new file mode 100644
index 0000000000000..f685bcd7d9d90
--- /dev/null
+++ b/db/schema_migrations/20210325152011
@@ -0,0 +1 @@
+379fdb3c52e55b51ebdb4a3b1e67c12f19b15e97cce22eed351e33953e389c85
\ No newline at end of file
diff --git a/db/structure.sql b/db/structure.sql
index 0fbbd4ea008fa..5a48efd8e44f9 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -10672,8 +10672,16 @@ CREATE TABLE ci_pipeline_artifacts (
file_format smallint NOT NULL,
file text,
expire_at timestamp with time zone,
+ verification_started_at timestamp with time zone,
+ verification_retry_at timestamp with time zone,
+ verified_at timestamp with time zone,
+ verification_state smallint DEFAULT 0 NOT NULL,
+ verification_retry_count smallint,
+ verification_checksum bytea,
+ verification_failure text,
CONSTRAINT check_191b5850ec CHECK ((char_length(file) <= 255)),
- CONSTRAINT check_abeeb71caf CHECK ((file IS NOT NULL))
+ CONSTRAINT check_abeeb71caf CHECK ((file IS NOT NULL)),
+ CONSTRAINT ci_pipeline_artifacts_verification_failure_text_limit CHECK ((char_length(verification_failure) <= 255))
);
CREATE SEQUENCE ci_pipeline_artifacts_id_seq
@@ -22158,6 +22166,10 @@ CREATE UNIQUE INDEX index_ci_job_variables_on_key_and_job_id ON ci_job_variables
CREATE UNIQUE INDEX index_ci_namespace_monthly_usages_on_namespace_id_and_date ON ci_namespace_monthly_usages USING btree (namespace_id, date);
+CREATE INDEX index_ci_pipeline_artifacts_failed_verification ON ci_pipeline_artifacts USING btree (verification_retry_at NULLS FIRST) WHERE (verification_state = 3);
+
+CREATE INDEX index_ci_pipeline_artifacts_needs_verification ON ci_pipeline_artifacts USING btree (verification_state) WHERE ((verification_state = 0) OR (verification_state = 3));
+
CREATE INDEX index_ci_pipeline_artifacts_on_expire_at ON ci_pipeline_artifacts USING btree (expire_at);
CREATE INDEX index_ci_pipeline_artifacts_on_pipeline_id ON ci_pipeline_artifacts USING btree (pipeline_id);
@@ -22166,6 +22178,10 @@ CREATE UNIQUE INDEX index_ci_pipeline_artifacts_on_pipeline_id_and_file_type ON
CREATE INDEX index_ci_pipeline_artifacts_on_project_id ON ci_pipeline_artifacts USING btree (project_id);
+CREATE INDEX index_ci_pipeline_artifacts_pending_verification ON ci_pipeline_artifacts USING btree (verified_at NULLS FIRST) WHERE (verification_state = 0);
+
+CREATE INDEX index_ci_pipeline_artifacts_verification_state ON ci_pipeline_artifacts USING btree (verification_state);
+
CREATE INDEX index_ci_pipeline_chat_data_on_chat_name_id ON ci_pipeline_chat_data USING btree (chat_name_id);
CREATE UNIQUE INDEX index_ci_pipeline_chat_data_on_pipeline_id ON ci_pipeline_chat_data USING btree (pipeline_id);
diff --git a/ee/db/geo/migrate/20210325150435_create_pipeline_artifact_registry.rb b/ee/db/geo/migrate/20210325150435_create_pipeline_artifact_registry.rb
new file mode 100644
index 0000000000000..7ed5d99cdd275
--- /dev/null
+++ b/ee/db/geo/migrate/20210325150435_create_pipeline_artifact_registry.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+class CreatePipelineArtifactRegistry < ActiveRecord::Migration[6.0]
+ include Gitlab::Database::MigrationHelpers
+
+ DOWNTIME = false
+
+ disable_ddl_transaction!
+
+ def up
+ create_table :pipeline_artifact_registry, id: :bigserial, force: :cascade do |t|
+ t.bigint :pipeline_artifact_id, null: false
+ t.datetime_with_timezone :created_at, null: false
+ t.datetime_with_timezone :last_synced_at
+ t.datetime_with_timezone :retry_at
+ t.datetime_with_timezone :verified_at
+ t.datetime_with_timezone :verification_started_at
+ t.datetime_with_timezone :verification_retry_at
+ t.integer :state, default: 0, null: false, limit: 2
+ t.integer :verification_state, default: 0, null: false, limit: 2
+ t.integer :retry_count, default: 0, limit: 2
+ t.integer :verification_retry_count, default: 0, limit: 2
+ t.boolean :checksum_mismatch, default: false, null: false
+ t.binary :verification_checksum
+ t.binary :verification_checksum_mismatched
+ t.string :verification_failure, limit: 255 # rubocop:disable Migration/PreventStrings
+ t.string :last_sync_failure, limit: 255 # rubocop:disable Migration/PreventStrings
+
+ t.index :pipeline_artifact_id, name: :index_pipeline_artifact_registry_on_pipeline_artifact_id, unique: true
+ t.index :retry_at
+ t.index :state
+ t.index :verification_retry_at, name: :pipeline_artifact_registry_failed_verification, order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 3))"
+ t.index :verification_state, name: :pipeline_artifact_registry_needs_verification, where: "((state = 2) AND (verification_state = ANY (ARRAY[0, 3])))"
+ t.index :verified_at, name: :pipeline_artifact_registry_pending_verification, order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 0))"
+ end
+ end
+
+ def down
+ drop_table :pipeline_artifact_registry
+ end
+end
diff --git a/ee/db/geo/schema.rb b/ee/db/geo/schema.rb
index 2781d2340170a..afbb8d83dd1ca 100644
--- a/ee/db/geo/schema.rb
+++ b/ee/db/geo/schema.rb
@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 2021_03_13_051642) do
+ActiveRecord::Schema.define(version: 2021_03_25_150435) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
@@ -160,7 +160,7 @@
t.integer "verification_state", limit: 2, default: 0, null: false
t.integer "retry_count", limit: 2, default: 0
t.integer "verification_retry_count", limit: 2, default: 0
- t.boolean "checksum_mismatch"
+ t.boolean "checksum_mismatch", default: false, null: false
t.binary "verification_checksum"
t.binary "verification_checksum_mismatched"
t.string "verification_failure", limit: 255
--
GitLab