diff --git a/changelogs/unreleased/mo-add-pipeline-artifact-geo-database.yml b/changelogs/unreleased/mo-add-pipeline-artifact-geo-database.yml new file mode 100644 index 0000000000000000000000000000000000000000..2fd629aa3ad0dab0ae536bb7fc4d17a4ff9b107b --- /dev/null +++ b/changelogs/unreleased/mo-add-pipeline-artifact-geo-database.yml @@ -0,0 +1,5 @@ +--- +title: Add geo database changes for pipeline artifact replication +merge_request: 57506 +author: +type: added diff --git a/db/migrate/20210325150837_add_verification_state_to_ci_pipeline_artifact.rb b/db/migrate/20210325150837_add_verification_state_to_ci_pipeline_artifact.rb new file mode 100644 index 0000000000000000000000000000000000000000..f6c506e59b78940bb0208876afc29c6b306fb197 --- /dev/null +++ b/db/migrate/20210325150837_add_verification_state_to_ci_pipeline_artifact.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class AddVerificationStateToCiPipelineArtifact < ActiveRecord::Migration[6.0] + DOWNTIME = false + + def change + change_table(:ci_pipeline_artifacts, bulk: true) do |t| + t.column :verification_started_at, :datetime_with_timezone + t.column :verification_retry_at, :datetime_with_timezone + t.column :verified_at, :datetime_with_timezone + t.integer :verification_state, default: 0, limit: 2, null: false + t.integer :verification_retry_count, limit: 2 + t.binary :verification_checksum, using: 'verification_checksum::bytea' + + t.text :verification_failure # rubocop:disable Migration/AddLimitToTextColumns + end + end +end diff --git a/db/migrate/20210325151758_add_verification_failure_limit_to_ci_pipeline_artifact.rb b/db/migrate/20210325151758_add_verification_failure_limit_to_ci_pipeline_artifact.rb new file mode 100644 index 0000000000000000000000000000000000000000..bc4dee2181faa73ec06db9fb1ed79d2936370963 --- /dev/null +++ b/db/migrate/20210325151758_add_verification_failure_limit_to_ci_pipeline_artifact.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class AddVerificationFailureLimitToCiPipelineArtifact < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + CONSTRAINT_NAME = 'ci_pipeline_artifacts_verification_failure_text_limit' + + def up + add_text_limit :ci_pipeline_artifacts, :verification_failure, 255, constraint_name: CONSTRAINT_NAME + end + + def down + remove_check_constraint(:ci_pipeline_artifacts, CONSTRAINT_NAME) + end +end diff --git a/db/migrate/20210325152011_add_verification_indexes_to_ci_pipeline_artifacts.rb b/db/migrate/20210325152011_add_verification_indexes_to_ci_pipeline_artifacts.rb new file mode 100644 index 0000000000000000000000000000000000000000..0822aee35a428eef2a2a64343cf802fb509aaf7c --- /dev/null +++ b/db/migrate/20210325152011_add_verification_indexes_to_ci_pipeline_artifacts.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class AddVerificationIndexesToCiPipelineArtifacts < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + VERIFICATION_STATE_INDEX_NAME = "index_ci_pipeline_artifacts_verification_state" + PENDING_VERIFICATION_INDEX_NAME = "index_ci_pipeline_artifacts_pending_verification" + FAILED_VERIFICATION_INDEX_NAME = "index_ci_pipeline_artifacts_failed_verification" + NEEDS_VERIFICATION_INDEX_NAME = "index_ci_pipeline_artifacts_needs_verification" + + disable_ddl_transaction! + + def up + add_concurrent_index :ci_pipeline_artifacts, :verification_state, name: VERIFICATION_STATE_INDEX_NAME + add_concurrent_index :ci_pipeline_artifacts, :verified_at, where: "(verification_state = 0)", order: { verified_at: 'ASC NULLS FIRST' }, name: PENDING_VERIFICATION_INDEX_NAME + add_concurrent_index :ci_pipeline_artifacts, :verification_retry_at, where: "(verification_state = 3)", order: { verification_retry_at: 'ASC NULLS FIRST' }, name: FAILED_VERIFICATION_INDEX_NAME + add_concurrent_index :ci_pipeline_artifacts, :verification_state, where: "(verification_state = 0 OR verification_state = 3)", name: NEEDS_VERIFICATION_INDEX_NAME + end + + def down + remove_concurrent_index_by_name :ci_pipeline_artifacts, VERIFICATION_STATE_INDEX_NAME + remove_concurrent_index_by_name :ci_pipeline_artifacts, PENDING_VERIFICATION_INDEX_NAME + remove_concurrent_index_by_name :ci_pipeline_artifacts, FAILED_VERIFICATION_INDEX_NAME + remove_concurrent_index_by_name :ci_pipeline_artifacts, NEEDS_VERIFICATION_INDEX_NAME + end +end diff --git a/db/schema_migrations/20210325150837 b/db/schema_migrations/20210325150837 new file mode 100644 index 0000000000000000000000000000000000000000..716462ad1871183a24128e5adab0942e51573921 --- /dev/null +++ b/db/schema_migrations/20210325150837 @@ -0,0 +1 @@ +6022464130d7a5697f52b9238837c6a6d3363fd349cbcb14052ff52de6ea2e59 \ No newline at end of file diff --git a/db/schema_migrations/20210325151758 b/db/schema_migrations/20210325151758 new file mode 100644 index 0000000000000000000000000000000000000000..a3d5f1b1b0a7f3200681bdbadfc150c7a2d3795b --- /dev/null +++ b/db/schema_migrations/20210325151758 @@ -0,0 +1 @@ +de55a114773961e6cae9ebae36ac93e60676555fe4c2973527511bb3a2eae69d \ No newline at end of file diff --git a/db/schema_migrations/20210325152011 b/db/schema_migrations/20210325152011 new file mode 100644 index 0000000000000000000000000000000000000000..f685bcd7d9d90987de19b885adacc0c28ec77e2d --- /dev/null +++ b/db/schema_migrations/20210325152011 @@ -0,0 +1 @@ +379fdb3c52e55b51ebdb4a3b1e67c12f19b15e97cce22eed351e33953e389c85 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 0fbbd4ea008faf2a5e6135b30870fbecd8381488..5a48efd8e44f9eeaf276e8f47b5240495dc9c8d3 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -10672,8 +10672,16 @@ CREATE TABLE ci_pipeline_artifacts ( file_format smallint NOT NULL, file text, expire_at timestamp with time zone, + verification_started_at timestamp with time zone, + verification_retry_at timestamp with time zone, + verified_at timestamp with time zone, + verification_state smallint DEFAULT 0 NOT NULL, + verification_retry_count smallint, + verification_checksum bytea, + verification_failure text, CONSTRAINT check_191b5850ec CHECK ((char_length(file) <= 255)), - CONSTRAINT check_abeeb71caf CHECK ((file IS NOT NULL)) + CONSTRAINT check_abeeb71caf CHECK ((file IS NOT NULL)), + CONSTRAINT ci_pipeline_artifacts_verification_failure_text_limit CHECK ((char_length(verification_failure) <= 255)) ); CREATE SEQUENCE ci_pipeline_artifacts_id_seq @@ -22158,6 +22166,10 @@ CREATE UNIQUE INDEX index_ci_job_variables_on_key_and_job_id ON ci_job_variables CREATE UNIQUE INDEX index_ci_namespace_monthly_usages_on_namespace_id_and_date ON ci_namespace_monthly_usages USING btree (namespace_id, date); +CREATE INDEX index_ci_pipeline_artifacts_failed_verification ON ci_pipeline_artifacts USING btree (verification_retry_at NULLS FIRST) WHERE (verification_state = 3); + +CREATE INDEX index_ci_pipeline_artifacts_needs_verification ON ci_pipeline_artifacts USING btree (verification_state) WHERE ((verification_state = 0) OR (verification_state = 3)); + CREATE INDEX index_ci_pipeline_artifacts_on_expire_at ON ci_pipeline_artifacts USING btree (expire_at); CREATE INDEX index_ci_pipeline_artifacts_on_pipeline_id ON ci_pipeline_artifacts USING btree (pipeline_id); @@ -22166,6 +22178,10 @@ CREATE UNIQUE INDEX index_ci_pipeline_artifacts_on_pipeline_id_and_file_type ON CREATE INDEX index_ci_pipeline_artifacts_on_project_id ON ci_pipeline_artifacts USING btree (project_id); +CREATE INDEX index_ci_pipeline_artifacts_pending_verification ON ci_pipeline_artifacts USING btree (verified_at NULLS FIRST) WHERE (verification_state = 0); + +CREATE INDEX index_ci_pipeline_artifacts_verification_state ON ci_pipeline_artifacts USING btree (verification_state); + CREATE INDEX index_ci_pipeline_chat_data_on_chat_name_id ON ci_pipeline_chat_data USING btree (chat_name_id); CREATE UNIQUE INDEX index_ci_pipeline_chat_data_on_pipeline_id ON ci_pipeline_chat_data USING btree (pipeline_id); diff --git a/ee/db/geo/migrate/20210325150435_create_pipeline_artifact_registry.rb b/ee/db/geo/migrate/20210325150435_create_pipeline_artifact_registry.rb new file mode 100644 index 0000000000000000000000000000000000000000..7ed5d99cdd2752f4a696aa787f4a4ea62ed7a1a7 --- /dev/null +++ b/ee/db/geo/migrate/20210325150435_create_pipeline_artifact_registry.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +class CreatePipelineArtifactRegistry < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + create_table :pipeline_artifact_registry, id: :bigserial, force: :cascade do |t| + t.bigint :pipeline_artifact_id, null: false + t.datetime_with_timezone :created_at, null: false + t.datetime_with_timezone :last_synced_at + t.datetime_with_timezone :retry_at + t.datetime_with_timezone :verified_at + t.datetime_with_timezone :verification_started_at + t.datetime_with_timezone :verification_retry_at + t.integer :state, default: 0, null: false, limit: 2 + t.integer :verification_state, default: 0, null: false, limit: 2 + t.integer :retry_count, default: 0, limit: 2 + t.integer :verification_retry_count, default: 0, limit: 2 + t.boolean :checksum_mismatch, default: false, null: false + t.binary :verification_checksum + t.binary :verification_checksum_mismatched + t.string :verification_failure, limit: 255 # rubocop:disable Migration/PreventStrings + t.string :last_sync_failure, limit: 255 # rubocop:disable Migration/PreventStrings + + t.index :pipeline_artifact_id, name: :index_pipeline_artifact_registry_on_pipeline_artifact_id, unique: true + t.index :retry_at + t.index :state + t.index :verification_retry_at, name: :pipeline_artifact_registry_failed_verification, order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 3))" + t.index :verification_state, name: :pipeline_artifact_registry_needs_verification, where: "((state = 2) AND (verification_state = ANY (ARRAY[0, 3])))" + t.index :verified_at, name: :pipeline_artifact_registry_pending_verification, order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 0))" + end + end + + def down + drop_table :pipeline_artifact_registry + end +end diff --git a/ee/db/geo/schema.rb b/ee/db/geo/schema.rb index 2781d2340170a4f2f2ecd0ca7aac9023f837c4f1..afbb8d83dd1ca08ea20e1e1393e44bbfc0f43888 100644 --- a/ee/db/geo/schema.rb +++ b/ee/db/geo/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_03_13_051642) do +ActiveRecord::Schema.define(version: 2021_03_25_150435) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -160,7 +160,7 @@ t.integer "verification_state", limit: 2, default: 0, null: false t.integer "retry_count", limit: 2, default: 0 t.integer "verification_retry_count", limit: 2, default: 0 - t.boolean "checksum_mismatch" + t.boolean "checksum_mismatch", default: false, null: false t.binary "verification_checksum" t.binary "verification_checksum_mismatched" t.string "verification_failure", limit: 255