Prepare group wiki registry table for verification

Changelog: changed
EE: true

db/docs/group_wiki_repository_states.yml

+---
+table_name: group_wiki_repository_states
+description: Separate table for group wiki verification states
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/126753
+milestone: '16.3'
+feature_categories:
+- geo_replication
+classes:
+- Geo::GroupWikiRepositoryState
+gitlab_schema: gitlab_main

db/migrate/20230717200940_create_group_wiki_repository_states.rb

+# frozen_string_literal: true
+
+class CreateGroupWikiRepositoryStates < Gitlab::Database::Migration[2.1]
+  VERIFICATION_STATE_INDEX_NAME = "index_group_wiki_repository_states_on_verification_state"
+  PENDING_VERIFICATION_INDEX_NAME = "index_group_wiki_repository_states_pending_verification"
+  FAILED_VERIFICATION_INDEX_NAME = "index_group_wiki_repository_states_failed_verification"
+  NEEDS_VERIFICATION_INDEX_NAME = "index_group_wiki_repository_states_needs_verification"
+
+  enable_lock_retries!
+
+  def up
+    create_table :group_wiki_repository_states do |t|
+      t.datetime_with_timezone :verification_started_at
+      t.datetime_with_timezone :verification_retry_at
+      t.datetime_with_timezone :verified_at
+      t.references :group_wiki_repository,
+        null: false,
+        index: { unique: true },
+        foreign_key: { primary_key: :group_id, on_delete: :cascade }
+      t.integer :verification_state, default: 0, limit: 2, null: false
+      t.integer :verification_retry_count, default: 0, limit: 2, null: false
+      t.binary :verification_checksum, using: 'verification_checksum::bytea'
+      t.text :verification_failure, limit: 255
+
+      t.index :verification_state, name: VERIFICATION_STATE_INDEX_NAME
+      t.index :verified_at,
+        where: "(verification_state = 0)",
+        order: { verified_at: 'ASC NULLS FIRST' },
+        name: PENDING_VERIFICATION_INDEX_NAME
+      t.index :verification_retry_at,
+        where: "(verification_state = 3)",
+        order: { verification_retry_at: 'ASC NULLS FIRST' },
+        name: FAILED_VERIFICATION_INDEX_NAME
+      t.index :verification_state,
+        where: "(verification_state = 0 OR verification_state = 3)",
+        name: NEEDS_VERIFICATION_INDEX_NAME
+    end
+  end
+
+  def down
+    drop_table :group_wiki_repository_states
+  end
+end

db/schema_migrations/20230717200940

+970898c369a13483eab732969f1ef3a63f7bcaf041fdb8a2b8c10e93da0de7c1
\ No newline at end of file

db/structure.sql

@@ -16740,6 +16740,28 @@ CREATE TABLE group_wiki_repositories (
     CONSTRAINT check_07f1c81806 CHECK ((char_length(disk_path) <= 80))
 );
+CREATE TABLE group_wiki_repository_states (
+    id bigint NOT NULL,
+    verification_started_at timestamp with time zone,
+    verification_retry_at timestamp with time zone,
+    verified_at timestamp with time zone,
+    group_wiki_repository_id bigint NOT NULL,
+    verification_state smallint DEFAULT 0 NOT NULL,
+    verification_retry_count smallint DEFAULT 0 NOT NULL,
+    verification_checksum bytea,
+    verification_failure text,
+    CONSTRAINT check_14d288436d CHECK ((char_length(verification_failure) <= 255))
+);
+
+CREATE SEQUENCE group_wiki_repository_states_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE group_wiki_repository_states_id_seq OWNED BY group_wiki_repository_states.id;
+
 CREATE TABLE historical_data (
     id integer NOT NULL,
     date date,
@@ -25449,6 +25471,8 @@ ALTER TABLE ONLY group_import_states ALTER COLUMN group_id SET DEFAULT nextval('
 ALTER TABLE ONLY group_repository_storage_moves ALTER COLUMN id SET DEFAULT nextval('group_repository_storage_moves_id_seq'::regclass);
+ALTER TABLE ONLY group_wiki_repository_states ALTER COLUMN id SET DEFAULT nextval('group_wiki_repository_states_id_seq'::regclass);
+
 ALTER TABLE ONLY historical_data ALTER COLUMN id SET DEFAULT nextval('historical_data_id_seq'::regclass);
 ALTER TABLE ONLY identities ALTER COLUMN id SET DEFAULT nextval('identities_id_seq'::regclass);
@@ -27557,6 +27581,9 @@ ALTER TABLE ONLY group_repository_storage_moves
 ALTER TABLE ONLY group_wiki_repositories
     ADD CONSTRAINT group_wiki_repositories_pkey PRIMARY KEY (group_id);
+ALTER TABLE ONLY group_wiki_repository_states
+    ADD CONSTRAINT group_wiki_repository_states_pkey PRIMARY KEY (id);
+
 ALTER TABLE ONLY historical_data
     ADD CONSTRAINT historical_data_pkey PRIMARY KEY (id);
@@ -31420,6 +31447,16 @@ CREATE UNIQUE INDEX index_group_wiki_repositories_on_disk_path ON group_wiki_rep
 CREATE INDEX index_group_wiki_repositories_on_shard_id ON group_wiki_repositories USING btree (shard_id);
+CREATE INDEX index_group_wiki_repository_states_failed_verification ON group_wiki_repository_states USING btree (verification_retry_at NULLS FIRST) WHERE (verification_state = 3);
+
+CREATE INDEX index_group_wiki_repository_states_needs_verification ON group_wiki_repository_states USING btree (verification_state) WHERE ((verification_state = 0) OR (verification_state = 3));
+
+CREATE UNIQUE INDEX index_group_wiki_repository_states_on_group_wiki_repository_id ON group_wiki_repository_states USING btree (group_wiki_repository_id);
+
+CREATE INDEX index_group_wiki_repository_states_on_verification_state ON group_wiki_repository_states USING btree (verification_state);
+
+CREATE INDEX index_group_wiki_repository_states_pending_verification ON group_wiki_repository_states USING btree (verified_at NULLS FIRST) WHERE (verification_state = 0);
+
 CREATE INDEX index_groups_on_parent_id_id ON namespaces USING btree (parent_id, id) WHERE ((type)::text = 'Group'::text);
 CREATE INDEX index_groups_on_path_and_id ON namespaces USING btree (path, id) WHERE ((type)::text = 'Group'::text);
@@ -37288,6 +37325,9 @@ ALTER TABLE ONLY required_code_owners_sections
 ALTER TABLE ONLY namespace_ldap_settings
     ADD CONSTRAINT fk_rails_82cd0ad4bb FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE;
+ALTER TABLE ONLY group_wiki_repository_states
+    ADD CONSTRAINT fk_rails_832511c9f1 FOREIGN KEY (group_wiki_repository_id) REFERENCES group_wiki_repositories(group_id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY cluster_enabled_grants
     ADD CONSTRAINT fk_rails_8336ce35af FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE;

ee/db/geo/migrate/20230717195110_prepare_group_wiki_repository_registry_for_verification.rb

+# frozen_string_literal: true
+
+class PrepareGroupWikiRepositoryRegistryForVerification < Gitlab::Database::Migration[2.1]
+  def up
+    add_column :group_wiki_repository_registry, :verified_at, :datetime_with_timezone
+    add_column :group_wiki_repository_registry, :verification_started_at, :datetime_with_timezone
+    add_column :group_wiki_repository_registry, :verification_retry_at, :datetime_with_timezone
+    add_column :group_wiki_repository_registry, :verification_state, :integer, default: 0, null: false, limit: 2
+    add_column :group_wiki_repository_registry, :verification_retry_count, :integer, default: 0, limit: 2, null: false
+    add_column :group_wiki_repository_registry, :checksum_mismatch, :boolean, default: false, null: false
+    add_column :group_wiki_repository_registry, :verification_checksum, :binary
+    add_column :group_wiki_repository_registry, :verification_checksum_mismatched, :binary
+
+    # rubocop:disable Migration/AddLimitToTextColumns
+    # The limit is added in a separate migration
+    add_column :group_wiki_repository_registry, :verification_failure, :text
+    # rubocop:enable Migration/AddLimitToTextColumns
+  end
+
+  def down
+    remove_column :group_wiki_repository_registry, :verified_at
+    remove_column :group_wiki_repository_registry, :verification_started_at
+    remove_column :group_wiki_repository_registry, :verification_retry_at
+    remove_column :group_wiki_repository_registry, :verification_state
+    remove_column :group_wiki_repository_registry, :verification_retry_count
+    remove_column :group_wiki_repository_registry, :checksum_mismatch
+    remove_column :group_wiki_repository_registry, :verification_checksum
+    remove_column :group_wiki_repository_registry, :verification_checksum_mismatched
+    remove_column :group_wiki_repository_registry, :verification_failure
+  end
+end

ee/db/geo/migrate/20230717195815_create_group_wiki_repository_registry_verification_indexies.rb

+# frozen_string_literal: true
+
+class CreateGroupWikiRepositoryRegistryVerificationIndexies < Gitlab::Database::Migration[2.1]
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :group_wiki_repository_registry,
+      :verification_retry_at,
+      name: :group_wiki_repository_registry_failed_verification,
+      order: 'NULLS FIRST', where: 'verification_state = 3'
+    add_concurrent_index :group_wiki_repository_registry,
+      :verification_state,
+      name: :group_wiki_repository_registry_needs_verification,
+      where: "verification_state = ANY (ARRAY[0, 3])"
+    add_concurrent_index :group_wiki_repository_registry,
+      :verified_at,
+      name: :group_wiki_repository_registry_pending_verification,
+      order: 'NULLS FIRST', where: 'verification_state = 0'
+  end
+
+  def down
+    remove_concurrent_index :group_wiki_repository_registry,
+      :verification_retry_at,
+      name: :group_wiki_repository_registry_failed_verification
+    remove_concurrent_index :group_wiki_repository_registry,
+      :verification_state,
+      name: :group_wiki_repository_registry_needs_verification
+    remove_concurrent_index :group_wiki_repository_registry,
+      :verified_at,
+      name: :container_repository_registry_pending_verification
+  end
+end

ee/db/geo/migrate/20230717200028_add_text_limit_to_group_wiki_repository_registry_verification_failure.rb

+# frozen_string_literal: true
+
+class AddTextLimitToGroupWikiRepositoryRegistryVerificationFailure < Gitlab::Database::Migration[2.1]
+  disable_ddl_transaction!
+
+  def up
+    add_text_limit :group_wiki_repository_registry, :verification_failure, 255
+  end
+
+  def down
+    remove_text_limit :group_wiki_repository_registry, :verification_failure
+  end
+end

ee/db/geo/schema_migrations/20230717195110

+923665cd91e041e6e6a36674bc86e0d76a176602a667031120f280de72232bb0
\ No newline at end of file

ee/db/geo/schema_migrations/20230717195815

+175b0e9113fb28059b8c0a420d2e1ae6d60f5d06c14faae037999136ea6c1eb5
\ No newline at end of file

ee/db/geo/schema_migrations/20230717200028

+3aef53bdb149463a69045e65204fb9a685403b1b0ceb268d0d1f18beb0bf9261
\ No newline at end of file

ee/db/geo/structure.sql

@@ -242,7 +242,17 @@ CREATE TABLE group_wiki_repository_registry (
     retry_count smallint DEFAULT 0,
     last_sync_failure text,
     force_to_redownload boolean,
-    missing_on_primary boolean
+    missing_on_primary boolean,
+    verified_at timestamp with time zone,
+    verification_started_at timestamp with time zone,
+    verification_retry_at timestamp with time zone,
+    verification_state smallint DEFAULT 0 NOT NULL,
+    verification_retry_count smallint DEFAULT 0 NOT NULL,
+    checksum_mismatch boolean DEFAULT false NOT NULL,
+    verification_checksum bytea,
+    verification_checksum_mismatched bytea,
+    verification_failure text,
+    CONSTRAINT check_0a6e7bc04a CHECK ((char_length(verification_failure) <= 255))
 );
 
 CREATE SEQUENCE group_wiki_repository_registry_id_seq
@@ -786,6 +796,12 @@ CREATE INDEX file_registry_needs_verification ON file_registry USING btree (veri
 
 CREATE INDEX file_registry_pending_verification ON file_registry USING btree (verified_at NULLS FIRST) WHERE ((state = 2) AND (verification_state = 0));
 
+CREATE INDEX group_wiki_repository_registry_failed_verification ON group_wiki_repository_registry USING btree (verification_retry_at NULLS FIRST) WHERE (verification_state = 3);
+
+CREATE INDEX group_wiki_repository_registry_needs_verification ON group_wiki_repository_registry USING btree (verification_state) WHERE (verification_state = ANY (ARRAY[0, 3]));
+
+CREATE INDEX group_wiki_repository_registry_pending_verification ON group_wiki_repository_registry USING btree (verified_at NULLS FIRST) WHERE (verification_state = 0);
+
 CREATE UNIQUE INDEX i_dependency_proxy_blob_registry_on_dependency_proxy_blob_id ON dependency_proxy_blob_registry USING btree (dependency_proxy_blob_id);
 
 CREATE INDEX idx_project_registry_failed_repositories_partial ON project_registry USING btree (repository_retry_count) WHERE ((repository_retry_count > 0) OR (last_repository_verification_failure IS NOT NULL) OR repository_checksum_mismatch);