From a7e7a8303af57ed28ca095719bfff1190693d30e Mon Sep 17 00:00:00 2001
From: Scott Murray <smurray@gitlab.com>
Date: Wed, 12 Feb 2025 00:19:08 +0000
Subject: [PATCH] Create new Geo verification state table for SnippetRepository
Relates to https://gitlab.com/gitlab-org/gitlab/-/issues/516947
Changelog: changed
EE: true
---
db/docs/snippet_repository_states.yml | 12 ++++++
...124417_create_snippet_repository_states.rb | 34 ++++++++++++++++
...to_snippet_repository_states_snippet_id.rb | 21 ++++++++++
db/schema_migrations/20250206124417 | 1 +
db/schema_migrations/20250213211743 | 1 +
db/structure.sql | 40 +++++++++++++++++++
ee/app/models/ee/snippet_repository.rb | 6 +++
ee/app/models/geo/snippet_repository_state.rb | 12 ++++++
.../geo/snippet_repository_states.rb | 15 +++++++
.../geo/snippet_repository_state_spec.rb | 14 +++++++
ee/spec/models/snippet_repository_spec.rb | 9 +++++
11 files changed, 165 insertions(+)
create mode 100644 db/docs/snippet_repository_states.yml
create mode 100644 db/migrate/20250206124417_create_snippet_repository_states.rb
create mode 100644 db/migrate/20250213211743_add_foreign_key_to_snippet_repository_states_snippet_id.rb
create mode 100644 db/schema_migrations/20250206124417
create mode 100644 db/schema_migrations/20250213211743
create mode 100644 ee/app/models/geo/snippet_repository_state.rb
create mode 100644 ee/spec/factories/geo/snippet_repository_states.rb
create mode 100644 ee/spec/models/geo/snippet_repository_state_spec.rb
diff --git a/db/docs/snippet_repository_states.yml b/db/docs/snippet_repository_states.yml
new file mode 100644
index 000000000000..49d762a14c64
--- /dev/null
+++ b/db/docs/snippet_repository_states.yml
@@ -0,0 +1,12 @@
+---
+table_name: snippet_repository_states
+classes:
+- Geo::SnippetRepositoryState
+feature_categories:
+- geo_replication
+description: Separate table for snippet repositories containing Geo verification metadata.
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/181255
+milestone: '17.10'
+gitlab_schema: gitlab_main_cell
+exempt_from_sharding: true
+table_size: small
diff --git a/db/migrate/20250206124417_create_snippet_repository_states.rb b/db/migrate/20250206124417_create_snippet_repository_states.rb
new file mode 100644
index 000000000000..49eb382f236c
--- /dev/null
+++ b/db/migrate/20250206124417_create_snippet_repository_states.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+class CreateSnippetRepositoryStates < Gitlab::Database::Migration[2.2]
+ milestone '17.10'
+
+ def change
+ create_table :snippet_repository_states do |t|
+ t.datetime_with_timezone :verification_started_at
+ t.datetime_with_timezone :verification_retry_at
+ t.datetime_with_timezone :verified_at
+ t.bigint :snippet_repository_id, null: false
+
+ t.integer :verification_state, default: 0, limit: 2, null: false
+ t.integer :verification_retry_count, default: 0, limit: 2
+
+ t.binary :verification_checksum, using: 'verification_checksum::bytea'
+ t.text :verification_failure, limit: 255
+
+ t.index :snippet_repository_id, unique: true
+ t.index :verification_state, name: 'index_snippet_repository_states_on_verification_state'
+ t.index :verified_at,
+ where: "(verification_state = 0)",
+ order: { verified_at: 'ASC NULLS FIRST' },
+ name: 'index_snippet_repository_states_pending_verification'
+ t.index :verification_retry_at,
+ where: "(verification_state = 3)",
+ order: { verification_retry_at: 'ASC NULLS FIRST' },
+ name: 'index_snippet_repository_states_failed_verification'
+ t.index :verification_state,
+ where: "(verification_state = 0 OR verification_state = 3)",
+ name: 'index_snippet_repository_states_needs_verification'
+ end
+ end
+end
diff --git a/db/migrate/20250213211743_add_foreign_key_to_snippet_repository_states_snippet_id.rb b/db/migrate/20250213211743_add_foreign_key_to_snippet_repository_states_snippet_id.rb
new file mode 100644
index 000000000000..8c18156413b0
--- /dev/null
+++ b/db/migrate/20250213211743_add_foreign_key_to_snippet_repository_states_snippet_id.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class AddForeignKeyToSnippetRepositoryStatesSnippetId < Gitlab::Database::Migration[2.2]
+ disable_ddl_transaction!
+
+ milestone '17.10'
+
+ def up
+ add_concurrent_foreign_key :snippet_repository_states,
+ :snippet_repositories,
+ column: :snippet_repository_id,
+ target_column: :snippet_id,
+ on_delete: :cascade
+ end
+
+ def down
+ with_lock_retries do
+ remove_foreign_key :snippet_repository_states, column: :snippet_repository_id
+ end
+ end
+end
diff --git a/db/schema_migrations/20250206124417 b/db/schema_migrations/20250206124417
new file mode 100644
index 000000000000..0178a85b0038
--- /dev/null
+++ b/db/schema_migrations/20250206124417
@@ -0,0 +1 @@
+5addc61acbabe1bcd3bb74ab87ea4291546a6e693352d630e77006dee873f5c6
\ No newline at end of file
diff --git a/db/schema_migrations/20250213211743 b/db/schema_migrations/20250213211743
new file mode 100644
index 000000000000..ce5f59bc3aae
--- /dev/null
+++ b/db/schema_migrations/20250213211743
@@ -0,0 +1 @@
+270e4a1ec37e327924489391588b5447005787dbcf88a295fd200895b100a40a
\ No newline at end of file
diff --git a/db/structure.sql b/db/structure.sql
index 9fb6101a663b..328340063805 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -21628,6 +21628,28 @@ CREATE TABLE snippet_repositories (
CONSTRAINT snippet_repositories_verification_failure_text_limit CHECK ((char_length(verification_failure) <= 255))
);
+CREATE TABLE snippet_repository_states (
+ id bigint NOT NULL,
+ verification_started_at timestamp with time zone,
+ verification_retry_at timestamp with time zone,
+ verified_at timestamp with time zone,
+ snippet_repository_id bigint NOT NULL,
+ verification_state smallint DEFAULT 0 NOT NULL,
+ verification_retry_count smallint DEFAULT 0,
+ verification_checksum bytea,
+ verification_failure text,
+ CONSTRAINT check_0dabaefb7f CHECK ((char_length(verification_failure) <= 255))
+);
+
+CREATE SEQUENCE snippet_repository_states_id_seq
+ START WITH 1
+ INCREMENT BY 1
+ NO MINVALUE
+ NO MAXVALUE
+ CACHE 1;
+
+ALTER SEQUENCE snippet_repository_states_id_seq OWNED BY snippet_repository_states.id;
+
CREATE TABLE snippet_repository_storage_moves (
id bigint NOT NULL,
created_at timestamp with time zone NOT NULL,
@@ -26086,6 +26108,8 @@ ALTER TABLE ONLY slack_integrations_scopes ALTER COLUMN id SET DEFAULT nextval('
ALTER TABLE ONLY smartcard_identities ALTER COLUMN id SET DEFAULT nextval('smartcard_identities_id_seq'::regclass);
+ALTER TABLE ONLY snippet_repository_states ALTER COLUMN id SET DEFAULT nextval('snippet_repository_states_id_seq'::regclass);
+
ALTER TABLE ONLY snippet_repository_storage_moves ALTER COLUMN id SET DEFAULT nextval('snippet_repository_storage_moves_id_seq'::regclass);
ALTER TABLE ONLY snippet_user_mentions ALTER COLUMN id SET DEFAULT nextval('snippet_user_mentions_id_seq'::regclass);
@@ -29008,6 +29032,9 @@ ALTER TABLE ONLY smartcard_identities
ALTER TABLE ONLY snippet_repositories
ADD CONSTRAINT snippet_repositories_pkey PRIMARY KEY (snippet_id);
+ALTER TABLE ONLY snippet_repository_states
+ ADD CONSTRAINT snippet_repository_states_pkey PRIMARY KEY (id);
+
ALTER TABLE ONLY snippet_repository_storage_moves
ADD CONSTRAINT snippet_repository_storage_moves_pkey PRIMARY KEY (id);
@@ -35068,6 +35095,16 @@ CREATE INDEX index_snippet_repositories_pending_verification ON snippet_reposito
CREATE INDEX index_snippet_repositories_verification_state ON snippet_repositories USING btree (verification_state);
+CREATE INDEX index_snippet_repository_states_failed_verification ON snippet_repository_states USING btree (verification_retry_at NULLS FIRST) WHERE (verification_state = 3);
+
+CREATE INDEX index_snippet_repository_states_needs_verification ON snippet_repository_states USING btree (verification_state) WHERE ((verification_state = 0) OR (verification_state = 3));
+
+CREATE UNIQUE INDEX index_snippet_repository_states_on_snippet_repository_id ON snippet_repository_states USING btree (snippet_repository_id);
+
+CREATE INDEX index_snippet_repository_states_on_verification_state ON snippet_repository_states USING btree (verification_state);
+
+CREATE INDEX index_snippet_repository_states_pending_verification ON snippet_repository_states USING btree (verified_at NULLS FIRST) WHERE (verification_state = 0);
+
CREATE INDEX index_snippet_repository_storage_moves_on_snippet_id ON snippet_repository_storage_moves USING btree (snippet_id);
CREATE INDEX index_snippet_repository_storage_moves_on_snippet_organization_ ON snippet_repository_storage_moves USING btree (snippet_organization_id);
@@ -39429,6 +39466,9 @@ ALTER TABLE ONLY csv_issue_imports
ALTER TABLE ONLY milestone_releases
ADD CONSTRAINT fk_5e73b8cad2 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE;
+ALTER TABLE ONLY snippet_repository_states
+ ADD CONSTRAINT fk_5f750f3182 FOREIGN KEY (snippet_repository_id) REFERENCES snippet_repositories(snippet_id) ON DELETE CASCADE;
+
ALTER TABLE ONLY packages_conan_package_revisions
ADD CONSTRAINT fk_5f7c6a9244 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE;
diff --git a/ee/app/models/ee/snippet_repository.rb b/ee/app/models/ee/snippet_repository.rb
index 4598d23677a6..7e9d78aea8cf 100644
--- a/ee/app/models/ee/snippet_repository.rb
+++ b/ee/app/models/ee/snippet_repository.rb
@@ -14,6 +14,12 @@ module SnippetRepository
include ::Gitlab::SQL::Pattern
with_replicator ::Geo::SnippetRepositoryReplicator
+
+ has_one :snippet_repository_state,
+ autosave: false,
+ inverse_of: :snippet_repository,
+ foreign_key: :snippet_repository_id,
+ class_name: '::Geo::SnippetRepositoryState'
end
class_methods do
diff --git a/ee/app/models/geo/snippet_repository_state.rb b/ee/app/models/geo/snippet_repository_state.rb
new file mode 100644
index 000000000000..883e0d75c61b
--- /dev/null
+++ b/ee/app/models/geo/snippet_repository_state.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Geo
+ class SnippetRepositoryState < ApplicationRecord
+ include ::Geo::VerificationStateDefinition
+
+ belongs_to :snippet_repository,
+ inverse_of: :snippet_repository_state
+
+ validates :verification_state, :snippet_repository, presence: true
+ end
+end
diff --git a/ee/spec/factories/geo/snippet_repository_states.rb b/ee/spec/factories/geo/snippet_repository_states.rb
new file mode 100644
index 000000000000..2fc9a60fbad5
--- /dev/null
+++ b/ee/spec/factories/geo/snippet_repository_states.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+ factory :geo_snippet_repository_state, class: 'Geo::SnippetRepositoryState' do
+ snippet_repository
+
+ trait(:checksummed) do
+ verification_checksum { 'abc' }
+ end
+
+ trait(:checksum_failure) do
+ verification_failure { 'Could not calculate the checksum' }
+ end
+ end
+end
diff --git a/ee/spec/models/geo/snippet_repository_state_spec.rb b/ee/spec/models/geo/snippet_repository_state_spec.rb
new file mode 100644
index 000000000000..5f3ba48b83d0
--- /dev/null
+++ b/ee/spec/models/geo/snippet_repository_state_spec.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Geo::SnippetRepositoryState, :geo, type: :model, feature_category: :geo_replication do
+ describe 'associations' do
+ it { is_expected.to belong_to(:snippet_repository).inverse_of(:snippet_repository_state) }
+ end
+
+ describe 'validations' do
+ it { is_expected.to validate_presence_of(:verification_state) }
+ it { is_expected.to validate_presence_of(:snippet_repository) }
+ end
+end
diff --git a/ee/spec/models/snippet_repository_spec.rb b/ee/spec/models/snippet_repository_spec.rb
index 05c550c15c2e..6d7c2a6ab959 100644
--- a/ee/spec/models/snippet_repository_spec.rb
+++ b/ee/spec/models/snippet_repository_spec.rb
@@ -11,6 +11,15 @@
stub_current_geo_node(secondary)
end
+ describe 'associations' do
+ it do
+ is_expected.to have_one(:snippet_repository_state)
+ .class_name('Geo::SnippetRepositoryState')
+ .with_foreign_key(:snippet_repository_id)
+ .inverse_of(:snippet_repository)
+ end
+ end
+
context 'with 3 groups, 2 projects, and 5 snippets' do
let(:group_1) { create(:group) }
let(:group_2) { create(:group) }
--
GitLab