From 031387143675d9acb7464c90c63e07ad6079de7c Mon Sep 17 00:00:00 2001
From: Matt Kasa <mkasa@gitlab.com>
Date: Wed, 18 Oct 2023 19:57:01 -0700
Subject: [PATCH] Add todos config for multiple partition scan detector
Relates to https://gitlab.com/gitlab-org/gitlab/-/issues/428704
---
scripts/database/query_analyzers.rb | 14 +++++++++++++-
scripts/database/query_analyzers.yml | 11 +++++++++++
scripts/database/query_analyzers/base.rb | 4 +++-
.../multiple_partition_scan_detector.rb | 10 ++++------
4 files changed, 31 insertions(+), 8 deletions(-)
create mode 100644 scripts/database/query_analyzers.yml
diff --git a/scripts/database/query_analyzers.rb b/scripts/database/query_analyzers.rb
index 390851df81a47..89eae5e6f2cec 100644
--- a/scripts/database/query_analyzers.rb
+++ b/scripts/database/query_analyzers.rb
@@ -1,11 +1,17 @@
# frozen_string_literal: true
+require 'yaml'
+
class Database
class QueryAnalyzers
attr_reader :analyzers
def initialize
- @analyzers = ObjectSpace.each_object(::Class).select { |c| c < Base }.map(&:new)
+ config = YAML.safe_load_file(File.expand_path('query_analyzers.yml', __dir__))
+ @analyzers = self.class.all.map do |subclass|
+ subclass_name = subclass.to_s.split('::').last
+ subclass.new(config[subclass_name])
+ end
end
def analyze(query)
@@ -15,6 +21,12 @@ def analyze(query)
def save!
analyzers.each(&:save!)
end
+
+ class << self
+ def all
+ ObjectSpace.each_object(::Class).select { |c| c < Base }
+ end
+ end
end
end
diff --git a/scripts/database/query_analyzers.yml b/scripts/database/query_analyzers.yml
new file mode 100644
index 0000000000000..8f87d96500dff
--- /dev/null
+++ b/scripts/database/query_analyzers.yml
@@ -0,0 +1,11 @@
+MultiplePartitionScanDetector:
+ tables:
+ - p_ci_builds
+ - p_ci_builds_metadata
+ - p_ci_job_annotations
+ - p_ci_runner_machine_builds
+ todos:
+ # List query fingerprints which should be ignored until they are fixed.
+ # These fingerprints can be found in the auto_explain pipeline artifacts.
+ # Example:
+ # - c2cfe803a497101b
diff --git a/scripts/database/query_analyzers/base.rb b/scripts/database/query_analyzers/base.rb
index 4bf47a32da183..e56a61b3c396c 100644
--- a/scripts/database/query_analyzers/base.rb
+++ b/scripts/database/query_analyzers/base.rb
@@ -7,9 +7,11 @@ class Database
class QueryAnalyzers
class Base
attr_accessor :output
+ attr_reader :config
- def initialize
+ def initialize(config)
@output = {}
+ @config = config
end
def filename
diff --git a/scripts/database/query_analyzers/multiple_partition_scan_detector.rb b/scripts/database/query_analyzers/multiple_partition_scan_detector.rb
index 3afce51f87a8f..1a1415dd8f228 100644
--- a/scripts/database/query_analyzers/multiple_partition_scan_detector.rb
+++ b/scripts/database/query_analyzers/multiple_partition_scan_detector.rb
@@ -5,14 +5,12 @@
class Database
class QueryAnalyzers
class MultiplePartitionScanDetector < Database::QueryAnalyzers::Base
- TABLES = %w[
- p_ci_builds p_ci_builds_metadata p_ci_job_annotations p_ci_runner_machine_builds
- ].freeze
-
def analyze(query)
super
- TABLES.each do |table_name|
+ return if config['todos']&.include?(query['fingerprint'])
+
+ config['tables'].each do |table_name|
if query['query'].include?(table_name) && query['plan'].to_s.include?('"Subplans Removed"=>0')
(output[table_name] ||= []) << query
end
@@ -20,7 +18,7 @@ def analyze(query)
end
def save!
- TABLES.each do |table_name|
+ config['tables'].each do |table_name|
next unless output[table_name]
Zlib::GzipWriter.open(output_path("#{table_name}_multiple_partition_scans.ndjson")) do |file|
--
GitLab