diff --git a/config/initializers/postgres_partitioning.rb b/config/initializers/postgres_partitioning.rb
index f7a1ebbff86c09dc1c4e6dddaa712a50e70227eb..4157f2e79157b81fc8f0011286437d567637d7d2 100644
--- a/config/initializers/postgres_partitioning.rb
+++ b/config/initializers/postgres_partitioning.rb
@@ -106,4 +106,16 @@
]
)
+# Enable partition management for the backfill table during web_hook_logs partitioning.
+# This way new partitions will be created as the trigger syncs new rows across to this table.
+Gitlab::Database::Partitioning.register_tables(
+ [
+ {
+ limit_connection_names: %i[main],
+ table_name: 'web_hook_logs_daily',
+ partitioned_column: :created_at, strategy: :daily, retain_for: 14.days
+ }
+ ]
+)
+
Gitlab::Database::Partitioning.sync_partitions_ignore_db_error
diff --git a/db/docs/web_hook_logs_daily.yml b/db/docs/web_hook_logs_daily.yml
new file mode 100644
index 0000000000000000000000000000000000000000..f34200908780b98417343adc3101bcd88351a230
--- /dev/null
+++ b/db/docs/web_hook_logs_daily.yml
@@ -0,0 +1,12 @@
+---
+table_name: web_hook_logs_daily
+classes:
+- WebHookLog
+feature_categories:
+- integrations
+description: Webhooks logs data partitioned by day.
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/175379
+milestone: '17.8'
+gitlab_schema: gitlab_main_cell
+sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/463856
+table_size: small
diff --git a/db/migrate/20241217140211_create_daily_partitioned_web_hook_log_copy.rb b/db/migrate/20241217140211_create_daily_partitioned_web_hook_log_copy.rb
new file mode 100644
index 0000000000000000000000000000000000000000..4431f0c1231912ff8e1468727318f107c6dcb028
--- /dev/null
+++ b/db/migrate/20241217140211_create_daily_partitioned_web_hook_log_copy.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+class CreateDailyPartitionedWebHookLogCopy < Gitlab::Database::Migration[2.2]
+ disable_ddl_transaction!
+ milestone '17.8'
+
+ TABLE_NAME = :web_hook_logs_daily
+ SOURCE_TABLE_NAME = :web_hook_logs
+
+ def up
+ transaction do
+ execute(<<~SQL)
+ CREATE TABLE #{TABLE_NAME} (
+ LIKE #{SOURCE_TABLE_NAME} INCLUDING ALL EXCLUDING INDEXES,
+ PRIMARY KEY (id, created_at)
+ ) PARTITION BY RANGE (created_at);
+
+ CREATE TABLE IF NOT EXISTS #{partition_name(nil)}
+ PARTITION OF #{TABLE_NAME}
+ FOR VALUES FROM (MINVALUE) TO (\'#{current_date.prev_day}\');
+
+ CREATE TABLE IF NOT EXISTS #{partition_name(current_date.prev_day)}
+ PARTITION OF #{TABLE_NAME}
+ FOR VALUES FROM (\'#{current_date.prev_day}\') TO (\'#{current_date}\');
+
+ CREATE TABLE IF NOT EXISTS #{partition_name(current_date)}
+ PARTITION OF #{TABLE_NAME}
+ FOR VALUES FROM (\'#{current_date}\') TO (\'#{current_date.next_day}\');
+
+ CREATE TABLE IF NOT EXISTS #{partition_name(current_date.next_day)}
+ PARTITION OF #{TABLE_NAME}
+ FOR VALUES FROM (\'#{current_date.next_day}\') TO (\'#{current_date.next_day.next_day}\')
+ SQL
+ end
+ end
+
+ def down
+ execute(<<~SQL)
+ DROP TABLE #{TABLE_NAME}
+ SQL
+ end
+
+ private
+
+ def current_date
+ Date.current
+ end
+
+ def partition_name(date)
+ suffix = date&.strftime('%Y%m%d') || '00000000'
+ "gitlab_partitions_dynamic.#{TABLE_NAME}_#{suffix}"
+ end
+end
diff --git a/db/migrate/20241217140216_create_trigger_to_web_hook_logs.rb b/db/migrate/20241217140216_create_trigger_to_web_hook_logs.rb
new file mode 100644
index 0000000000000000000000000000000000000000..7e2b12b80bb0e91d360cb551da57fffd8a42f858
--- /dev/null
+++ b/db/migrate/20241217140216_create_trigger_to_web_hook_logs.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class CreateTriggerToWebHookLogs < Gitlab::Database::Migration[2.2]
+ include Gitlab::Database::SchemaHelpers
+ include Gitlab::Database::PartitioningMigrationHelpers::TableManagementHelpers
+
+ milestone '17.8'
+
+ SOURCE_TABLE = :web_hook_logs
+ TARGET_TABLE = :web_hook_logs_daily
+ UNIQUE_KEY = [:id, :created_at].freeze
+
+ def up
+ create_trigger_to_sync_tables(SOURCE_TABLE, TARGET_TABLE, UNIQUE_KEY)
+ end
+
+ def down
+ drop_sync_trigger(SOURCE_TABLE)
+ end
+end
diff --git a/db/schema_migrations/20241217140211 b/db/schema_migrations/20241217140211
new file mode 100644
index 0000000000000000000000000000000000000000..2d5a79f95bf633f469d38501e3bb496516b21905
--- /dev/null
+++ b/db/schema_migrations/20241217140211
@@ -0,0 +1 @@
+db476eb6b416a71e0b45bfa186e437ffdd30e1cb71e843aedf4de7c06ab65c48
\ No newline at end of file
diff --git a/db/schema_migrations/20241217140216 b/db/schema_migrations/20241217140216
new file mode 100644
index 0000000000000000000000000000000000000000..26dab8b09b0f030495dee140ddba3abe77890af2
--- /dev/null
+++ b/db/schema_migrations/20241217140216
@@ -0,0 +1 @@
+08e0001b798c0ad73bca409cdabf8c1dcb13099b4c08e0d49fa1c10051870393
\ No newline at end of file
diff --git a/db/structure.sql b/db/structure.sql
index 04ea6be00054daaa9ebd0ff93eebdfad2f3f2d26..752eed12e3de3a1f060a6fe5b0fde873a5d8966b 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -752,6 +752,64 @@ $$;
COMMENT ON FUNCTION table_sync_function_0992e728d3() IS 'Partitioning migration: table sync for merge_request_diff_commits table';
+CREATE FUNCTION table_sync_function_29bc99d6db() RETURNS trigger
+ LANGUAGE plpgsql
+ AS $$
+BEGIN
+IF (TG_OP = 'DELETE') THEN
+ DELETE FROM web_hook_logs_daily where "id" = OLD."id" AND "created_at" = OLD."created_at";
+ELSIF (TG_OP = 'UPDATE') THEN
+ UPDATE web_hook_logs_daily
+ SET "web_hook_id" = NEW."web_hook_id",
+ "trigger" = NEW."trigger",
+ "url" = NEW."url",
+ "request_headers" = NEW."request_headers",
+ "request_data" = NEW."request_data",
+ "response_headers" = NEW."response_headers",
+ "response_body" = NEW."response_body",
+ "response_status" = NEW."response_status",
+ "execution_duration" = NEW."execution_duration",
+ "internal_error_message" = NEW."internal_error_message",
+ "updated_at" = NEW."updated_at",
+ "url_hash" = NEW."url_hash"
+ WHERE web_hook_logs_daily."id" = NEW."id" AND web_hook_logs_daily."created_at" = NEW."created_at";
+ELSIF (TG_OP = 'INSERT') THEN
+ INSERT INTO web_hook_logs_daily ("id",
+ "web_hook_id",
+ "trigger",
+ "url",
+ "request_headers",
+ "request_data",
+ "response_headers",
+ "response_body",
+ "response_status",
+ "execution_duration",
+ "internal_error_message",
+ "updated_at",
+ "created_at",
+ "url_hash")
+ VALUES (NEW."id",
+ NEW."web_hook_id",
+ NEW."trigger",
+ NEW."url",
+ NEW."request_headers",
+ NEW."request_data",
+ NEW."response_headers",
+ NEW."response_body",
+ NEW."response_status",
+ NEW."execution_duration",
+ NEW."internal_error_message",
+ NEW."updated_at",
+ NEW."created_at",
+ NEW."url_hash");
+END IF;
+RETURN NULL;
+
+END
+$$;
+
+COMMENT ON FUNCTION table_sync_function_29bc99d6db() IS 'Partitioning migration: table sync for web_hook_logs table';
+
CREATE FUNCTION table_sync_function_3f39f64fc3() RETURNS trigger
LANGUAGE plpgsql
AS $$
@@ -3991,6 +4049,34 @@ CREATE TABLE web_hook_logs (
)
PARTITION BY RANGE (created_at);
+CREATE SEQUENCE web_hook_logs_id_seq
+ START WITH 1
+ INCREMENT BY 1
+ NO MINVALUE
+ NO MAXVALUE
+ CACHE 1;
+
+ALTER SEQUENCE web_hook_logs_id_seq OWNED BY web_hook_logs.id;
+
+CREATE TABLE web_hook_logs_daily (
+ id bigint DEFAULT nextval('web_hook_logs_id_seq'::regclass) NOT NULL,
+ web_hook_id bigint NOT NULL,
+ trigger character varying,
+ url character varying,
+ request_headers text,
+ request_data text,
+ response_headers text,
+ response_body text,
+ response_status character varying,
+ execution_duration double precision,
+ internal_error_message character varying,
+ updated_at timestamp without time zone NOT NULL,
+ created_at timestamp without time zone NOT NULL,
+ url_hash text,
+ CONSTRAINT check_df72cb58f5 CHECK ((char_length(url_hash) <= 44))
+)
+PARTITION BY RANGE (created_at);
+
CREATE TABLE zoekt_tasks (
id bigint NOT NULL,
partition_id bigint DEFAULT 1 NOT NULL,
@@ -22217,15 +22303,6 @@ CREATE SEQUENCE vulnerability_user_mentions_id_seq
ALTER SEQUENCE vulnerability_user_mentions_id_seq OWNED BY vulnerability_user_mentions.id;
-CREATE SEQUENCE web_hook_logs_id_seq
- START WITH 1
- INCREMENT BY 1
- NO MINVALUE
- NO MAXVALUE
- CACHE 1;
-
-ALTER SEQUENCE web_hook_logs_id_seq OWNED BY web_hook_logs.id;
-
CREATE TABLE web_hooks (
id bigint NOT NULL,
project_id bigint,
@@ -27436,6 +27513,9 @@ ALTER TABLE ONLY vulnerability_statistics
ALTER TABLE ONLY vulnerability_user_mentions
ADD CONSTRAINT vulnerability_user_mentions_pkey PRIMARY KEY (id);
+ALTER TABLE ONLY web_hook_logs_daily
+ ADD CONSTRAINT web_hook_logs_daily_pkey PRIMARY KEY (id, created_at);
+
ALTER TABLE ONLY web_hook_logs
ADD CONSTRAINT web_hook_logs_pkey PRIMARY KEY (id, created_at);
@@ -35743,6 +35823,8 @@ CREATE TRIGGER table_sync_trigger_57c8465cd7 AFTER INSERT OR DELETE OR UPDATE ON
CREATE TRIGGER table_sync_trigger_61879721b5 AFTER INSERT OR DELETE OR UPDATE ON ci_runners FOR EACH ROW EXECUTE FUNCTION table_sync_function_686d6c7993();
+CREATE TRIGGER table_sync_trigger_b99eb6998c AFTER INSERT OR DELETE OR UPDATE ON web_hook_logs FOR EACH ROW EXECUTE FUNCTION table_sync_function_29bc99d6db();
+
CREATE TRIGGER table_sync_trigger_bc3e7b56bd AFTER INSERT OR DELETE OR UPDATE ON ci_runner_machines FOR EACH ROW EXECUTE FUNCTION table_sync_function_e438f29263();
CREATE TRIGGER table_sync_trigger_cd362c20e2 AFTER INSERT OR DELETE OR UPDATE ON merge_request_diff_files FOR EACH ROW EXECUTE FUNCTION table_sync_function_3f39f64fc3();
diff --git a/spec/db/schema_spec.rb b/spec/db/schema_spec.rb
index 3456f9ce015d5c425faeb896f6198d42aa8c5e83..cef0e33f38c1f9f34af81539e8225eb31b0ea05f 100644
--- a/spec/db/schema_spec.rb
+++ b/spec/db/schema_spec.rb
@@ -233,6 +233,7 @@
# See: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/87584
# Fixes performance issues with the deletion of web-hooks with many log entries
web_hook_logs: %w[web_hook_id],
+ web_hook_logs_daily: %w[web_hook_id],
webauthn_registrations: %w[u2f_registration_id], # this column will be dropped
ml_candidates: %w[internal_id],
value_stream_dashboard_counts: %w[namespace_id],
diff --git a/spec/lib/gitlab/database/sharding_key_spec.rb b/spec/lib/gitlab/database/sharding_key_spec.rb
index 6600975cab1ae4e6c3156e84ede4e9c156906029..cfd40ecd8e45a44f9c9b9b0f2709cd9b997af2c5 100644
--- a/spec/lib/gitlab/database/sharding_key_spec.rb
+++ b/spec/lib/gitlab/database/sharding_key_spec.rb
@@ -14,7 +14,8 @@
'merge_request_diff_files_99208b8fac', # has a desired sharding key instead
'ml_model_metadata', # has a desired sharding key instead.
'p_ci_pipeline_variables', # has a desired sharding key instead
- 'sbom_occurrences_vulnerabilities' # has desired sharding key instead
+ 'sbom_occurrences_vulnerabilities', # has desired sharding key instead
+ 'web_hook_logs_daily' # temporary copy of web_hook_logs
]
end