From f8e6e55eb0cc83df0ff010511ed802324d657cd9 Mon Sep 17 00:00:00 2001 From: Manoj M J <mmj@gitlab.com> Date: Thu, 8 Feb 2024 14:52:19 +0100 Subject: [PATCH] Add desired sharding keys for vulnerability_management Add desired sharding keys for feature category `vulnerability_management`. These tables have been identified as a [cell local tables](https://docs.gitlab.com/ee/development/database/multiple_databases.html#guidelines-on-choosing-between-gitlab_main_cell-and-gitlab_main_clusterwide-schema). All cell local tables require a [sharding key](https://docs.gitlab.com/ee/development/database/multiple_databases.html#defining-a-sharding-key-for-all-cell-local-tables) or a [desired sharding key](https://docs.gitlab.com/ee/development/database/multiple_databases.html#defining-a-desired_sharding_key-for-automatically-backfilling-a-sharding_key) A desired sharding key has been automatically selected for these tables. These keys were chosen as the desired sharding keys because the table has a :belongs_to relationship to a table that itself has a `NOT NULL` sharding key. Additionally, `gitlab_schema` has been set to `gitlab_main_cell` for any tables didn't use this schema already. For these tables we have also added `allow_cross_joins`, `allow_cross_transactions` and `allow_cross_foreign_keys`. These will silence any existing violations, allowing the pipeline to pass without requiring further changes. In the future, we'll remove these `allow_...` statements and fix any violations as they arise. You can read more about this in the [documentation for multiple databases](https://docs.gitlab.com/ee/development/database/multiple_databases.html) We have assigned a random backend engineer from ~"group::threat insights" to review these changes. Please confirm that: - each of these tables can be classified as cell local - the selected desired sharding key is appropriate - the backfill configuration for the desired sharding key is correct When you are finished, please request a review from the database maintainer suggested by Danger. If you have any questions or concerns, reach out to @tigerwnz, @DylanGriffith or @manojmj. If you would like to go through similar merged MRs so as to gather an understanding on this topic, you can use [this](https://gitlab.com/gitlab-org/gitlab/-/merge_requests?scope=all&state=merged&label_name[]=automation%3Agitlab-housekeeper-authored) link. This change was generated by [gitlab-housekeeper](https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/gitlab-housekeeper) Changelog: other --- .../vulnerability_external_issue_links.yml | 20 +++++++++++++++++-- db/docs/vulnerability_finding_evidences.yml | 17 +++++++++++++++- db/docs/vulnerability_finding_links.yml | 17 +++++++++++++++- db/docs/vulnerability_finding_signatures.yml | 20 +++++++++++++++++-- db/docs/vulnerability_flags.yml | 20 +++++++++++++++++-- db/docs/vulnerability_issue_links.yml | 17 +++++++++++++++- db/docs/vulnerability_merge_request_links.yml | 17 +++++++++++++++- .../vulnerability_occurrence_identifiers.yml | 17 +++++++++++++++- .../vulnerability_occurrence_pipelines.yml | 17 +++++++++++++++- db/docs/vulnerability_state_transitions.yml | 17 +++++++++++++++- db/docs/vulnerability_user_mentions.yml | 17 +++++++++++++++- 11 files changed, 182 insertions(+), 14 deletions(-) diff --git a/db/docs/vulnerability_external_issue_links.yml b/db/docs/vulnerability_external_issue_links.yml index 4c2dcd8d8d6f0..12ac652b4c8a1 100644 --- a/db/docs/vulnerability_external_issue_links.yml +++ b/db/docs/vulnerability_external_issue_links.yml @@ -4,7 +4,23 @@ classes: - Vulnerabilities::ExternalIssueLink feature_categories: - vulnerability_management -description: Stores information about connections between external issue trackers and vulnerabilities +description: Stores information about connections between external issue trackers + and vulnerabilities introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/48465 milestone: '13.7' -gitlab_schema: gitlab_main +gitlab_schema: gitlab_main_cell +allow_cross_joins: +- gitlab_main_clusterwide +allow_cross_transactions: +- gitlab_main_clusterwide +allow_cross_foreign_keys: +- gitlab_main_clusterwide +desired_sharding_key: + project_id: + references: projects + backfill_via: + parent: + foreign_key: vulnerability_id + table: vulnerabilities + sharding_key: project_id + belongs_to: vulnerability diff --git a/db/docs/vulnerability_finding_evidences.yml b/db/docs/vulnerability_finding_evidences.yml index 35ecfd57fe3f5..4bb6bec600d11 100644 --- a/db/docs/vulnerability_finding_evidences.yml +++ b/db/docs/vulnerability_finding_evidences.yml @@ -7,4 +7,19 @@ feature_categories: description: Stores evidence used to identify presence of a vulnerability introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/56790 milestone: '13.11' -gitlab_schema: gitlab_main +gitlab_schema: gitlab_main_cell +allow_cross_joins: +- gitlab_main_clusterwide +allow_cross_transactions: +- gitlab_main_clusterwide +allow_cross_foreign_keys: +- gitlab_main_clusterwide +desired_sharding_key: + project_id: + references: projects + backfill_via: + parent: + foreign_key: vulnerability_occurrence_id + table: vulnerability_occurrences + sharding_key: project_id + belongs_to: finding diff --git a/db/docs/vulnerability_finding_links.yml b/db/docs/vulnerability_finding_links.yml index 267355dd87391..f848d17adddae 100644 --- a/db/docs/vulnerability_finding_links.yml +++ b/db/docs/vulnerability_finding_links.yml @@ -7,4 +7,19 @@ feature_categories: description: Stores URLs relevant to the vulnerability findings introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/46555 milestone: '13.6' -gitlab_schema: gitlab_main +gitlab_schema: gitlab_main_cell +allow_cross_joins: +- gitlab_main_clusterwide +allow_cross_transactions: +- gitlab_main_clusterwide +allow_cross_foreign_keys: +- gitlab_main_clusterwide +desired_sharding_key: + project_id: + references: projects + backfill_via: + parent: + foreign_key: vulnerability_occurrence_id + table: vulnerability_occurrences + sharding_key: project_id + belongs_to: finding diff --git a/db/docs/vulnerability_finding_signatures.yml b/db/docs/vulnerability_finding_signatures.yml index 9a1e59697cf52..3e4541502f818 100644 --- a/db/docs/vulnerability_finding_signatures.yml +++ b/db/docs/vulnerability_finding_signatures.yml @@ -4,7 +4,23 @@ classes: - Vulnerabilities::FindingSignature feature_categories: - vulnerability_management -description: Stores signatures of vulnerability locations which are used to improve tracking +description: Stores signatures of vulnerability locations which are used to improve + tracking introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/57840 milestone: '13.11' -gitlab_schema: gitlab_main +gitlab_schema: gitlab_main_cell +allow_cross_joins: +- gitlab_main_clusterwide +allow_cross_transactions: +- gitlab_main_clusterwide +allow_cross_foreign_keys: +- gitlab_main_clusterwide +desired_sharding_key: + project_id: + references: projects + backfill_via: + parent: + foreign_key: finding_id + table: vulnerability_occurrences + sharding_key: project_id + belongs_to: finding diff --git a/db/docs/vulnerability_flags.yml b/db/docs/vulnerability_flags.yml index fabc8a482000a..9b0071bf96bdb 100644 --- a/db/docs/vulnerability_flags.yml +++ b/db/docs/vulnerability_flags.yml @@ -4,7 +4,23 @@ classes: - Vulnerabilities::Flag feature_categories: - vulnerability_management -description: Stores additional information for vulnerabilities, for example if a vulnerability is identified as a false positive +description: Stores additional information for vulnerabilities, for example if a vulnerability + is identified as a false positive introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/65573 milestone: '14.1' -gitlab_schema: gitlab_main +gitlab_schema: gitlab_main_cell +allow_cross_joins: +- gitlab_main_clusterwide +allow_cross_transactions: +- gitlab_main_clusterwide +allow_cross_foreign_keys: +- gitlab_main_clusterwide +desired_sharding_key: + project_id: + references: projects + backfill_via: + parent: + foreign_key: vulnerability_occurrence_id + table: vulnerability_occurrences + sharding_key: project_id + belongs_to: finding diff --git a/db/docs/vulnerability_issue_links.yml b/db/docs/vulnerability_issue_links.yml index 4bbc587707e26..0a3441e7fae3c 100644 --- a/db/docs/vulnerability_issue_links.yml +++ b/db/docs/vulnerability_issue_links.yml @@ -7,4 +7,19 @@ feature_categories: description: Join table between Vulnerabilities and Issues introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/19852 milestone: '12.5' -gitlab_schema: gitlab_main +gitlab_schema: gitlab_main_cell +allow_cross_joins: +- gitlab_main_clusterwide +allow_cross_transactions: +- gitlab_main_clusterwide +allow_cross_foreign_keys: +- gitlab_main_clusterwide +desired_sharding_key: + project_id: + references: projects + backfill_via: + parent: + foreign_key: vulnerability_id + table: vulnerabilities + sharding_key: project_id + belongs_to: vulnerability diff --git a/db/docs/vulnerability_merge_request_links.yml b/db/docs/vulnerability_merge_request_links.yml index 8cc71b2a76af3..ce1e9a7a27f4f 100644 --- a/db/docs/vulnerability_merge_request_links.yml +++ b/db/docs/vulnerability_merge_request_links.yml @@ -7,4 +7,19 @@ feature_categories: description: Join table between Vulnerabilities and Merge Requests introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/92096 milestone: '15.2' -gitlab_schema: gitlab_main +gitlab_schema: gitlab_main_cell +allow_cross_joins: +- gitlab_main_clusterwide +allow_cross_transactions: +- gitlab_main_clusterwide +allow_cross_foreign_keys: +- gitlab_main_clusterwide +desired_sharding_key: + project_id: + references: projects + backfill_via: + parent: + foreign_key: vulnerability_id + table: vulnerabilities + sharding_key: project_id + belongs_to: vulnerability diff --git a/db/docs/vulnerability_occurrence_identifiers.yml b/db/docs/vulnerability_occurrence_identifiers.yml index cd2236631aa3e..1fe7cd1585f54 100644 --- a/db/docs/vulnerability_occurrence_identifiers.yml +++ b/db/docs/vulnerability_occurrence_identifiers.yml @@ -7,4 +7,19 @@ feature_categories: description: Join table between Findings and Identifiers introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/6896 milestone: '11.4' -gitlab_schema: gitlab_main +gitlab_schema: gitlab_main_cell +allow_cross_joins: +- gitlab_main_clusterwide +allow_cross_transactions: +- gitlab_main_clusterwide +allow_cross_foreign_keys: +- gitlab_main_clusterwide +desired_sharding_key: + project_id: + references: projects + backfill_via: + parent: + foreign_key: occurrence_id + table: vulnerability_occurrences + sharding_key: project_id + belongs_to: finding diff --git a/db/docs/vulnerability_occurrence_pipelines.yml b/db/docs/vulnerability_occurrence_pipelines.yml index 542d40268245b..6039fac8b15f7 100644 --- a/db/docs/vulnerability_occurrence_pipelines.yml +++ b/db/docs/vulnerability_occurrence_pipelines.yml @@ -7,4 +7,19 @@ feature_categories: description: Join table between Findings and Pipelines introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/7578 milestone: '11.5' -gitlab_schema: gitlab_main +gitlab_schema: gitlab_main_cell +allow_cross_joins: +- gitlab_main_clusterwide +allow_cross_transactions: +- gitlab_main_clusterwide +allow_cross_foreign_keys: +- gitlab_main_clusterwide +desired_sharding_key: + project_id: + references: projects + backfill_via: + parent: + foreign_key: occurrence_id + table: vulnerability_occurrences + sharding_key: project_id + belongs_to: finding diff --git a/db/docs/vulnerability_state_transitions.yml b/db/docs/vulnerability_state_transitions.yml index 1a669b1527d63..0e028ee6a573d 100644 --- a/db/docs/vulnerability_state_transitions.yml +++ b/db/docs/vulnerability_state_transitions.yml @@ -7,4 +7,19 @@ feature_categories: description: Stores state transitions of a Vulnerability introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/87957 milestone: '15.1' -gitlab_schema: gitlab_main +gitlab_schema: gitlab_main_cell +allow_cross_joins: +- gitlab_main_clusterwide +allow_cross_transactions: +- gitlab_main_clusterwide +allow_cross_foreign_keys: +- gitlab_main_clusterwide +desired_sharding_key: + project_id: + references: projects + backfill_via: + parent: + foreign_key: vulnerability_id + table: vulnerabilities + sharding_key: project_id + belongs_to: vulnerability diff --git a/db/docs/vulnerability_user_mentions.yml b/db/docs/vulnerability_user_mentions.yml index 9a95c83472618..296245fb2cc54 100644 --- a/db/docs/vulnerability_user_mentions.yml +++ b/db/docs/vulnerability_user_mentions.yml @@ -7,4 +7,19 @@ feature_categories: description: Stores notes for a given vulnerability introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/27515 milestone: '13.0' -gitlab_schema: gitlab_main +gitlab_schema: gitlab_main_cell +allow_cross_joins: +- gitlab_main_clusterwide +allow_cross_transactions: +- gitlab_main_clusterwide +allow_cross_foreign_keys: +- gitlab_main_clusterwide +desired_sharding_key: + project_id: + references: projects + backfill_via: + parent: + foreign_key: vulnerability_id + table: vulnerabilities + sharding_key: project_id + belongs_to: vulnerability -- GitLab