Merge branch 'secure-files-replication-db-changes' into 'master'

DB additions for CI Secure Files Replicator

See merge request gitlab-org/gitlab!90510

db/docs/ci_secure_file_states.yml

+---
+table_name: ci_secure_file_states
+classes:
+- Geo::CiSecureFileState
+feature_categories:
+- pipeline_authoring
+description: Stores verification state for Geo replicated Project-level Secure Files.
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/90510
+milestone: '15.2'

db/migrate/20220617141347_create_ci_secure_file_states.rb

+# frozen_string_literal: true
+# rubocop:disable Layout/LineLength
+
+class CreateCiSecureFileStates < Gitlab::Database::Migration[2.0]
+  VERIFICATION_STATE_INDEX_NAME = "index_ci_secure_file_states_on_verification_state"
+  PENDING_VERIFICATION_INDEX_NAME = "index_ci_secure_file_states_pending_verification"
+  FAILED_VERIFICATION_INDEX_NAME = "index_ci_secure_file_states_failed_verification"
+  NEEDS_VERIFICATION_INDEX_NAME = "index_ci_secure_file_states_needs_verification"
+
+  enable_lock_retries!
+
+  def up
+    create_table :ci_secure_file_states, id: false do |t|
+      t.datetime_with_timezone :verification_started_at
+      t.datetime_with_timezone :verification_retry_at
+      t.datetime_with_timezone :verified_at
+      t.references :ci_secure_file, primary_key: true, null: false, foreign_key: { on_delete: :cascade }
+      t.integer :verification_state, default: 0, limit: 2, null: false
+      t.integer :verification_retry_count, limit: 2
+      t.binary :verification_checksum, using: 'verification_checksum::bytea'
+      t.text :verification_failure, limit: 255
+
+      t.index :verification_state, name: VERIFICATION_STATE_INDEX_NAME
+      t.index :verified_at, where: "(verification_state = 0)", order: { verified_at: 'ASC NULLS FIRST' }, name: PENDING_VERIFICATION_INDEX_NAME
+      t.index :verification_retry_at, where: "(verification_state = 3)", order: { verification_retry_at: 'ASC NULLS FIRST' }, name: FAILED_VERIFICATION_INDEX_NAME
+      t.index :verification_state, where: "(verification_state = 0 OR verification_state = 3)", name: NEEDS_VERIFICATION_INDEX_NAME
+    end
+  end
+
+  def down
+    drop_table :ci_secure_file_states
+  end
+end
+# rubocop:enable Layout/LineLength

db/schema_migrations/20220617141347

+64473c05cc67d6d87d921921ea09e0962f51092d5bb565cc387def8d91676728
\ No newline at end of file

db/structure.sql

@@ -13122,6 +13122,27 @@ CREATE SEQUENCE ci_running_builds_id_seq
 ALTER SEQUENCE ci_running_builds_id_seq OWNED BY ci_running_builds.id;
+CREATE TABLE ci_secure_file_states (
+    verification_started_at timestamp with time zone,
+    verification_retry_at timestamp with time zone,
+    verified_at timestamp with time zone,
+    ci_secure_file_id bigint NOT NULL,
+    verification_state smallint DEFAULT 0 NOT NULL,
+    verification_retry_count smallint,
+    verification_checksum bytea,
+    verification_failure text,
+    CONSTRAINT check_a79e5a9261 CHECK ((char_length(verification_failure) <= 255))
+);
+
+CREATE SEQUENCE ci_secure_file_states_ci_secure_file_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE ci_secure_file_states_ci_secure_file_id_seq OWNED BY ci_secure_file_states.ci_secure_file_id;
+
 CREATE TABLE ci_secure_files (
     id bigint NOT NULL,
     project_id bigint NOT NULL,
@@ -22765,6 +22786,8 @@ ALTER TABLE ONLY ci_runners ALTER COLUMN id SET DEFAULT nextval('ci_runners_id_s
 ALTER TABLE ONLY ci_running_builds ALTER COLUMN id SET DEFAULT nextval('ci_running_builds_id_seq'::regclass);
+ALTER TABLE ONLY ci_secure_file_states ALTER COLUMN ci_secure_file_id SET DEFAULT nextval('ci_secure_file_states_ci_secure_file_id_seq'::regclass);
+
 ALTER TABLE ONLY ci_secure_files ALTER COLUMN id SET DEFAULT nextval('ci_secure_files_id_seq'::regclass);
 ALTER TABLE ONLY ci_sources_pipelines ALTER COLUMN id SET DEFAULT nextval('ci_sources_pipelines_id_seq'::regclass);
@@ -24496,6 +24519,9 @@ ALTER TABLE ONLY ci_runners
 ALTER TABLE ONLY ci_running_builds
     ADD CONSTRAINT ci_running_builds_pkey PRIMARY KEY (id);
+ALTER TABLE ONLY ci_secure_file_states
+    ADD CONSTRAINT ci_secure_file_states_pkey PRIMARY KEY (ci_secure_file_id);
+
 ALTER TABLE ONLY ci_secure_files
     ADD CONSTRAINT ci_secure_files_pkey PRIMARY KEY (id);
@@ -27571,6 +27597,16 @@ CREATE INDEX index_ci_running_builds_on_project_id ON ci_running_builds USING bt
 CREATE INDEX index_ci_running_builds_on_runner_id ON ci_running_builds USING btree (runner_id);
+CREATE INDEX index_ci_secure_file_states_failed_verification ON ci_secure_file_states USING btree (verification_retry_at NULLS FIRST) WHERE (verification_state = 3);
+
+CREATE INDEX index_ci_secure_file_states_needs_verification ON ci_secure_file_states USING btree (verification_state) WHERE ((verification_state = 0) OR (verification_state = 3));
+
+CREATE INDEX index_ci_secure_file_states_on_ci_secure_file_id ON ci_secure_file_states USING btree (ci_secure_file_id);
+
+CREATE INDEX index_ci_secure_file_states_on_verification_state ON ci_secure_file_states USING btree (verification_state);
+
+CREATE INDEX index_ci_secure_file_states_pending_verification ON ci_secure_file_states USING btree (verified_at NULLS FIRST) WHERE (verification_state = 0);
+
 CREATE INDEX index_ci_secure_files_on_project_id ON ci_secure_files USING btree (project_id);
 CREATE INDEX index_ci_sources_pipelines_on_pipeline_id ON ci_sources_pipelines USING btree (pipeline_id);
@@ -32870,6 +32906,9 @@ ALTER TABLE ONLY badges
 ALTER TABLE ONLY resource_label_events
     ADD CONSTRAINT fk_rails_5ac1d2fc24 FOREIGN KEY (issue_id) REFERENCES issues(id) ON DELETE CASCADE;
+ALTER TABLE ONLY ci_secure_file_states
+    ADD CONSTRAINT fk_rails_5adba40c5f FOREIGN KEY (ci_secure_file_id) REFERENCES ci_secure_files(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY approval_merge_request_rules_groups
     ADD CONSTRAINT fk_rails_5b2ecf6139 FOREIGN KEY (approval_merge_request_rule_id) REFERENCES approval_merge_request_rules(id) ON DELETE CASCADE;

ee/db/geo/migrate/20220617125507_create_ci_secure_file_registry.rb

+# frozen_string_literal: true
+# rubocop:disable Layout/LineLength
+
+class CreateCiSecureFileRegistry < Gitlab::Database::Migration[1.0]
+  disable_ddl_transaction!
+
+  def up
+    ActiveRecord::Base.transaction do
+      create_table :ci_secure_file_registry, id: :bigserial, force: :cascade do |t|
+        t.bigint :ci_secure_file_id, null: false
+        t.datetime_with_timezone :created_at, null: false
+        t.datetime_with_timezone :last_synced_at
+        t.datetime_with_timezone :retry_at
+        t.datetime_with_timezone :verified_at
+        t.datetime_with_timezone :verification_started_at
+        t.datetime_with_timezone :verification_retry_at
+        t.integer :state, default: 0, null: false, limit: 2
+        t.integer :verification_state, default: 0, null: false, limit: 2
+        t.integer :retry_count, default: 0, limit: 2, null: false
+        t.integer :verification_retry_count, default: 0, limit: 2, null: false
+        t.boolean :checksum_mismatch, default: false, null: false
+        t.binary :verification_checksum
+        t.binary :verification_checksum_mismatched
+        t.text :verification_failure, limit: 255
+        t.text :last_sync_failure, limit: 255
+
+        t.index :ci_secure_file_id, name: :index_ci_secure_file_registry_on_ci_secure_file_id, unique: true
+        t.index :retry_at
+        t.index :state
+        # To optimize performance of CiSecureFileRegistry.verification_failed_batch
+        t.index :verification_retry_at, name: :ci_secure_file_registry_failed_verification, order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 3))"
+        # To optimize performance of CiSecureFileRegistry.needs_verification_count
+        t.index :verification_state, name:  :ci_secure_file_registry_needs_verification, where: "((state = 2)  AND (verification_state = ANY (ARRAY[0, 3])))"
+        # To optimize performance of CiSecureFileRegistry.verification_pending_batch
+        t.index :verified_at, name: :ci_secure_file_registry_pending_verification, order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 0))"
+      end
+    end
+  end
+
+  def down
+    drop_table :ci_secure_file_registry
+  end
+end
+# rubocop:enable Layout/LineLength

ee/db/geo/schema_migrations/20220617125507

+c39dd7bf53264b620eae2ef0107599433ce6248cd24d005e2b906f22bb3999a9
\ No newline at end of file

ee/db/geo/structure.sql

@@ -5,6 +5,37 @@ CREATE TABLE ar_internal_metadata (
     updated_at timestamp(6) without time zone NOT NULL
 );
 
+CREATE TABLE ci_secure_file_registry (
+    id bigint NOT NULL,
+    ci_secure_file_id bigint NOT NULL,
+    created_at timestamp with time zone NOT NULL,
+    last_synced_at timestamp with time zone,
+    retry_at timestamp with time zone,
+    verified_at timestamp with time zone,
+    verification_started_at timestamp with time zone,
+    verification_retry_at timestamp with time zone,
+    state smallint DEFAULT 0 NOT NULL,
+    verification_state smallint DEFAULT 0 NOT NULL,
+    retry_count smallint DEFAULT 0 NOT NULL,
+    verification_retry_count smallint DEFAULT 0 NOT NULL,
+    checksum_mismatch boolean DEFAULT false NOT NULL,
+    verification_checksum bytea,
+    verification_checksum_mismatched bytea,
+    verification_failure text,
+    last_sync_failure text,
+    CONSTRAINT check_17bd5fc9fa CHECK ((char_length(last_sync_failure) <= 255)),
+    CONSTRAINT check_420f14e38c CHECK ((char_length(verification_failure) <= 255))
+);
+
+CREATE SEQUENCE ci_secure_file_registry_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE ci_secure_file_registry_id_seq OWNED BY ci_secure_file_registry.id;
+
 CREATE TABLE container_repository_registry (
     id integer NOT NULL,
     container_repository_id integer NOT NULL,
@@ -433,6 +464,8 @@ CREATE SEQUENCE terraform_state_version_registry_id_seq
 
 ALTER SEQUENCE terraform_state_version_registry_id_seq OWNED BY terraform_state_version_registry.id;
 
+ALTER TABLE ONLY ci_secure_file_registry ALTER COLUMN id SET DEFAULT nextval('ci_secure_file_registry_id_seq'::regclass);
+
 ALTER TABLE ONLY container_repository_registry ALTER COLUMN id SET DEFAULT nextval('container_repository_registry_id_seq'::regclass);
 
 ALTER TABLE ONLY design_registry ALTER COLUMN id SET DEFAULT nextval('design_registry_id_seq'::regclass);
@@ -466,6 +499,9 @@ ALTER TABLE ONLY terraform_state_version_registry ALTER COLUMN id SET DEFAULT ne
 ALTER TABLE ONLY ar_internal_metadata
     ADD CONSTRAINT ar_internal_metadata_pkey PRIMARY KEY (key);
 
+ALTER TABLE ONLY ci_secure_file_registry
+    ADD CONSTRAINT ci_secure_file_registry_pkey PRIMARY KEY (id);
+
 ALTER TABLE ONLY container_repository_registry
     ADD CONSTRAINT container_repository_registry_pkey PRIMARY KEY (id);
 
@@ -514,6 +550,12 @@ ALTER TABLE ONLY snippet_repository_registry
 ALTER TABLE ONLY terraform_state_version_registry
     ADD CONSTRAINT terraform_state_version_registry_pkey PRIMARY KEY (id);
 
+CREATE INDEX ci_secure_file_registry_failed_verification ON ci_secure_file_registry USING btree (verification_retry_at NULLS FIRST) WHERE ((state = 2) AND (verification_state = 3));
+
+CREATE INDEX ci_secure_file_registry_needs_verification ON ci_secure_file_registry USING btree (verification_state) WHERE ((state = 2) AND (verification_state = ANY (ARRAY[0, 3])));
+
+CREATE INDEX ci_secure_file_registry_pending_verification ON ci_secure_file_registry USING btree (verified_at NULLS FIRST) WHERE ((state = 2) AND (verification_state = 0));
+
 CREATE INDEX file_registry_failed_verification ON file_registry USING btree (verification_retry_at NULLS FIRST) WHERE ((state = 2) AND (verification_state = 3));
 
 CREATE INDEX file_registry_needs_verification ON file_registry USING btree (verification_state) WHERE ((state = 2) AND (verification_state = ANY (ARRAY[0, 3])));
@@ -542,6 +584,12 @@ CREATE INDEX idx_repository_checksum_mismatch ON project_registry USING btree (p
 
 CREATE INDEX idx_wiki_checksum_mismatch ON project_registry USING btree (project_id) WHERE (wiki_checksum_mismatch = true);
 
+CREATE UNIQUE INDEX index_ci_secure_file_registry_on_ci_secure_file_id ON ci_secure_file_registry USING btree (ci_secure_file_id);
+
+CREATE INDEX index_ci_secure_file_registry_on_retry_at ON ci_secure_file_registry USING btree (retry_at);
+
+CREATE INDEX index_ci_secure_file_registry_on_state ON ci_secure_file_registry USING btree (state);
+
 CREATE INDEX index_container_repository_registry_on_retry_at ON container_repository_registry USING btree (retry_at);
 
 CREATE INDEX index_container_repository_registry_on_state ON container_repository_registry USING btree (state);

lib/gitlab/database/gitlab_schemas.yml

@@ -112,6 +112,7 @@ ci_runners: :gitlab_ci
 ci_running_builds: :gitlab_ci
 ci_sources_pipelines: :gitlab_ci
 ci_secure_files: :gitlab_ci
+ci_secure_file_states: :gitlab_ci
 ci_sources_projects: :gitlab_ci
 ci_stages: :gitlab_ci
 ci_subscriptions_projects: :gitlab_ci