diff --git a/db/docs/snippet_repository_states.yml b/db/docs/snippet_repository_states.yml new file mode 100644 index 0000000000000000000000000000000000000000..49d762a14c647b842fa485b6c9f6302af6bc76d1 --- /dev/null +++ b/db/docs/snippet_repository_states.yml @@ -0,0 +1,12 @@ +--- +table_name: snippet_repository_states +classes: +- Geo::SnippetRepositoryState +feature_categories: +- geo_replication +description: Separate table for snippet repositories containing Geo verification metadata. +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/181255 +milestone: '17.10' +gitlab_schema: gitlab_main_cell +exempt_from_sharding: true +table_size: small diff --git a/db/migrate/20250206124417_create_snippet_repository_states.rb b/db/migrate/20250206124417_create_snippet_repository_states.rb new file mode 100644 index 0000000000000000000000000000000000000000..49eb382f236c340621ac5997472303ea3a1b7fe2 --- /dev/null +++ b/db/migrate/20250206124417_create_snippet_repository_states.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +class CreateSnippetRepositoryStates < Gitlab::Database::Migration[2.2] + milestone '17.10' + + def change + create_table :snippet_repository_states do |t| + t.datetime_with_timezone :verification_started_at + t.datetime_with_timezone :verification_retry_at + t.datetime_with_timezone :verified_at + t.bigint :snippet_repository_id, null: false + + t.integer :verification_state, default: 0, limit: 2, null: false + t.integer :verification_retry_count, default: 0, limit: 2 + + t.binary :verification_checksum, using: 'verification_checksum::bytea' + t.text :verification_failure, limit: 255 + + t.index :snippet_repository_id, unique: true + t.index :verification_state, name: 'index_snippet_repository_states_on_verification_state' + t.index :verified_at, + where: "(verification_state = 0)", + order: { verified_at: 'ASC NULLS FIRST' }, + name: 'index_snippet_repository_states_pending_verification' + t.index :verification_retry_at, + where: "(verification_state = 3)", + order: { verification_retry_at: 'ASC NULLS FIRST' }, + name: 'index_snippet_repository_states_failed_verification' + t.index :verification_state, + where: "(verification_state = 0 OR verification_state = 3)", + name: 'index_snippet_repository_states_needs_verification' + end + end +end diff --git a/db/migrate/20250213211743_add_foreign_key_to_snippet_repository_states_snippet_id.rb b/db/migrate/20250213211743_add_foreign_key_to_snippet_repository_states_snippet_id.rb new file mode 100644 index 0000000000000000000000000000000000000000..8c18156413b0216973153e9b739a05ea5f464624 --- /dev/null +++ b/db/migrate/20250213211743_add_foreign_key_to_snippet_repository_states_snippet_id.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class AddForeignKeyToSnippetRepositoryStatesSnippetId < Gitlab::Database::Migration[2.2] + disable_ddl_transaction! + + milestone '17.10' + + def up + add_concurrent_foreign_key :snippet_repository_states, + :snippet_repositories, + column: :snippet_repository_id, + target_column: :snippet_id, + on_delete: :cascade + end + + def down + with_lock_retries do + remove_foreign_key :snippet_repository_states, column: :snippet_repository_id + end + end +end diff --git a/db/schema_migrations/20250206124417 b/db/schema_migrations/20250206124417 new file mode 100644 index 0000000000000000000000000000000000000000..0178a85b0038f968174fb5c96702d22ca4052db1 --- /dev/null +++ b/db/schema_migrations/20250206124417 @@ -0,0 +1 @@ +5addc61acbabe1bcd3bb74ab87ea4291546a6e693352d630e77006dee873f5c6 \ No newline at end of file diff --git a/db/schema_migrations/20250213211743 b/db/schema_migrations/20250213211743 new file mode 100644 index 0000000000000000000000000000000000000000..ce5f59bc3aaefb544a1f6e01c6d0c9614dd74968 --- /dev/null +++ b/db/schema_migrations/20250213211743 @@ -0,0 +1 @@ +270e4a1ec37e327924489391588b5447005787dbcf88a295fd200895b100a40a \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 9fb6101a663be69b9f8eaadd5c545e7820f37d8c..328340063805b387df64a9d75f9110349273cdc2 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -21628,6 +21628,28 @@ CREATE TABLE snippet_repositories ( CONSTRAINT snippet_repositories_verification_failure_text_limit CHECK ((char_length(verification_failure) <= 255)) ); +CREATE TABLE snippet_repository_states ( + id bigint NOT NULL, + verification_started_at timestamp with time zone, + verification_retry_at timestamp with time zone, + verified_at timestamp with time zone, + snippet_repository_id bigint NOT NULL, + verification_state smallint DEFAULT 0 NOT NULL, + verification_retry_count smallint DEFAULT 0, + verification_checksum bytea, + verification_failure text, + CONSTRAINT check_0dabaefb7f CHECK ((char_length(verification_failure) <= 255)) +); + +CREATE SEQUENCE snippet_repository_states_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE snippet_repository_states_id_seq OWNED BY snippet_repository_states.id; + CREATE TABLE snippet_repository_storage_moves ( id bigint NOT NULL, created_at timestamp with time zone NOT NULL, @@ -26086,6 +26108,8 @@ ALTER TABLE ONLY slack_integrations_scopes ALTER COLUMN id SET DEFAULT nextval(' ALTER TABLE ONLY smartcard_identities ALTER COLUMN id SET DEFAULT nextval('smartcard_identities_id_seq'::regclass); +ALTER TABLE ONLY snippet_repository_states ALTER COLUMN id SET DEFAULT nextval('snippet_repository_states_id_seq'::regclass); + ALTER TABLE ONLY snippet_repository_storage_moves ALTER COLUMN id SET DEFAULT nextval('snippet_repository_storage_moves_id_seq'::regclass); ALTER TABLE ONLY snippet_user_mentions ALTER COLUMN id SET DEFAULT nextval('snippet_user_mentions_id_seq'::regclass); @@ -29008,6 +29032,9 @@ ALTER TABLE ONLY smartcard_identities ALTER TABLE ONLY snippet_repositories ADD CONSTRAINT snippet_repositories_pkey PRIMARY KEY (snippet_id); +ALTER TABLE ONLY snippet_repository_states + ADD CONSTRAINT snippet_repository_states_pkey PRIMARY KEY (id); + ALTER TABLE ONLY snippet_repository_storage_moves ADD CONSTRAINT snippet_repository_storage_moves_pkey PRIMARY KEY (id); @@ -35068,6 +35095,16 @@ CREATE INDEX index_snippet_repositories_pending_verification ON snippet_reposito CREATE INDEX index_snippet_repositories_verification_state ON snippet_repositories USING btree (verification_state); +CREATE INDEX index_snippet_repository_states_failed_verification ON snippet_repository_states USING btree (verification_retry_at NULLS FIRST) WHERE (verification_state = 3); + +CREATE INDEX index_snippet_repository_states_needs_verification ON snippet_repository_states USING btree (verification_state) WHERE ((verification_state = 0) OR (verification_state = 3)); + +CREATE UNIQUE INDEX index_snippet_repository_states_on_snippet_repository_id ON snippet_repository_states USING btree (snippet_repository_id); + +CREATE INDEX index_snippet_repository_states_on_verification_state ON snippet_repository_states USING btree (verification_state); + +CREATE INDEX index_snippet_repository_states_pending_verification ON snippet_repository_states USING btree (verified_at NULLS FIRST) WHERE (verification_state = 0); + CREATE INDEX index_snippet_repository_storage_moves_on_snippet_id ON snippet_repository_storage_moves USING btree (snippet_id); CREATE INDEX index_snippet_repository_storage_moves_on_snippet_organization_ ON snippet_repository_storage_moves USING btree (snippet_organization_id); @@ -39429,6 +39466,9 @@ ALTER TABLE ONLY csv_issue_imports ALTER TABLE ONLY milestone_releases ADD CONSTRAINT fk_5e73b8cad2 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; +ALTER TABLE ONLY snippet_repository_states + ADD CONSTRAINT fk_5f750f3182 FOREIGN KEY (snippet_repository_id) REFERENCES snippet_repositories(snippet_id) ON DELETE CASCADE; + ALTER TABLE ONLY packages_conan_package_revisions ADD CONSTRAINT fk_5f7c6a9244 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; diff --git a/ee/app/models/ee/snippet_repository.rb b/ee/app/models/ee/snippet_repository.rb index 4598d23677a61bb81bc18261a4fcbc133f0a184b..7e9d78aea8cfc7b5f1ef59078c189edbba54be7e 100644 --- a/ee/app/models/ee/snippet_repository.rb +++ b/ee/app/models/ee/snippet_repository.rb @@ -14,6 +14,12 @@ module SnippetRepository include ::Gitlab::SQL::Pattern with_replicator ::Geo::SnippetRepositoryReplicator + + has_one :snippet_repository_state, + autosave: false, + inverse_of: :snippet_repository, + foreign_key: :snippet_repository_id, + class_name: '::Geo::SnippetRepositoryState' end class_methods do diff --git a/ee/app/models/geo/snippet_repository_state.rb b/ee/app/models/geo/snippet_repository_state.rb new file mode 100644 index 0000000000000000000000000000000000000000..883e0d75c61b29df0e8c7b1dfa12a0ef145a35df --- /dev/null +++ b/ee/app/models/geo/snippet_repository_state.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Geo + class SnippetRepositoryState < ApplicationRecord + include ::Geo::VerificationStateDefinition + + belongs_to :snippet_repository, + inverse_of: :snippet_repository_state + + validates :verification_state, :snippet_repository, presence: true + end +end diff --git a/ee/spec/factories/geo/snippet_repository_states.rb b/ee/spec/factories/geo/snippet_repository_states.rb new file mode 100644 index 0000000000000000000000000000000000000000..2fc9a60fbad58ecef275b49c78e6714db243df01 --- /dev/null +++ b/ee/spec/factories/geo/snippet_repository_states.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :geo_snippet_repository_state, class: 'Geo::SnippetRepositoryState' do + snippet_repository + + trait(:checksummed) do + verification_checksum { 'abc' } + end + + trait(:checksum_failure) do + verification_failure { 'Could not calculate the checksum' } + end + end +end diff --git a/ee/spec/models/geo/snippet_repository_state_spec.rb b/ee/spec/models/geo/snippet_repository_state_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..5f3ba48b83d046313ab15b0e2ed25038add85ad8 --- /dev/null +++ b/ee/spec/models/geo/snippet_repository_state_spec.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Geo::SnippetRepositoryState, :geo, type: :model, feature_category: :geo_replication do + describe 'associations' do + it { is_expected.to belong_to(:snippet_repository).inverse_of(:snippet_repository_state) } + end + + describe 'validations' do + it { is_expected.to validate_presence_of(:verification_state) } + it { is_expected.to validate_presence_of(:snippet_repository) } + end +end diff --git a/ee/spec/models/snippet_repository_spec.rb b/ee/spec/models/snippet_repository_spec.rb index 05c550c15c2eadbf4f0a99c9f55b442072210095..6d7c2a6ab959ce722d63ba1d13542f57e6be9088 100644 --- a/ee/spec/models/snippet_repository_spec.rb +++ b/ee/spec/models/snippet_repository_spec.rb @@ -11,6 +11,15 @@ stub_current_geo_node(secondary) end + describe 'associations' do + it do + is_expected.to have_one(:snippet_repository_state) + .class_name('Geo::SnippetRepositoryState') + .with_foreign_key(:snippet_repository_id) + .inverse_of(:snippet_repository) + end + end + context 'with 3 groups, 2 projects, and 5 snippets' do let(:group_1) { create(:group) } let(:group_2) { create(:group) }