diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index ef2056f164c857b339085001121317ff1dce15d7..7aa115718c779fbf60461e1976c52fc7fd8f658c 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -573,11 +573,8 @@ rspec:merge-auto-explain-logs: - .rails:rules:rspec-merge-auto-explain-logs stage: post-test needs: !reference ["rspec:coverage", "needs"] - before_script: - - source scripts/utils.sh - - source scripts/rspec_helpers.sh script: - - merge_auto_explain_logs + - scripts/merge-auto-explain-logs artifacts: name: auto-explain-logs expire_in: 31d diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index 97def7091c4dd8331e9b22630f714bd355de61ce..c4525bf160edc581d5b7003462ab3539d57b41c3 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -95,9 +95,6 @@ .if-merge-request-labels-skip-undercoverage: &if-merge-request-labels-skip-undercoverage if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:skip-undercoverage/' -.if-merge-request-labels-record-queries: &if-merge-request-labels-record-queries - if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:record-queries/' - .if-merge-request-labels-jh-contribution: &if-merge-request-labels-jh-contribution if: '$CI_MERGE_REQUEST_LABELS =~ /JiHu contribution/' @@ -2290,7 +2287,8 @@ - <<: *if-merge-request-labels-pipeline-expedite when: never - <<: *if-merge-request-labels-run-all-rspec - - <<: *if-merge-request-labels-record-queries + - <<: *if-merge-request + changes: *code-backstage-patterns - <<: *if-default-branch-refs changes: *code-patterns diff --git a/scripts/merge-auto-explain-logs b/scripts/merge-auto-explain-logs index 0dff4fb33f8062c9492c071f679bccaeb57dd0d2..23ddf65be69eb7f4d32fd0cf1ab8893755f957b8 100755 --- a/scripts/merge-auto-explain-logs +++ b/scripts/merge-auto-explain-logs @@ -1,14 +1,41 @@ #!/usr/bin/env ruby # frozen_string_literal: true -require "set" -require "json" +require 'json' +require 'set' +require 'zlib' + +source = ENV['CI_MERGE_REQUEST_SOURCE_BRANCH_SHA'] +target = ENV['CI_MERGE_REQUEST_TARGET_BRANCH_SHA'] +log_path = ENV['RSPEC_AUTO_EXPLAIN_LOG_PATH'] +logs_path = File.dirname(log_path) + +exit(0) unless Dir.exist?(logs_path) fingerprints = Set.new +jobs = Set.new + +JOB_NAME = %r{^(.*)\.\d+\.[^.]+\.ndjson\.gz$} + +Zlib::GzipWriter.open(log_path) do |log| + Dir[File.join(logs_path, '*.gz')].reject { |p| p == log_path }.each do |file| + job_name = File.basename(file)[JOB_NAME, 1] + Zlib::GzipReader.open(file) do |gz| + gz.each_line do |line| + json = JSON.parse(line) + fingerprint = json['fingerprint'] + + next unless fingerprints.add?(fingerprint) + + jobs << job_name + json['job_name'] = job_name + log.puts(json.to_s) + end + end -ARGF.each_line do |line| - fingerprint = JSON.parse(line)['fingerprint'] - $stdout.puts(line) && $stdout.flush if fingerprints.add?(fingerprint) + File.delete(file) + end end -warn("auto_explain log contains #{fingerprints.size} entries") +warn("auto_explain log contains #{fingerprints.size} entries from: #{jobs.to_a.sort.join(', ')}") +warn("auto_explain comparison of #{target} to #{source}") if source && target diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh index 46ffbc223ebcafdb516bee77b19928160f1aad72..6c39f126afa6cbb4836833b940d982189de2c275 100644 --- a/scripts/rspec_helpers.sh +++ b/scripts/rspec_helpers.sh @@ -486,13 +486,3 @@ function is_rspec_last_run_results_file_missing() { return 1 fi } - -function merge_auto_explain_logs() { - local auto_explain_logs_path="$(dirname "${RSPEC_AUTO_EXPLAIN_LOG_PATH}")/" - - for file in ${auto_explain_logs_path}*.gz; do - (gunzip -c "${file}" && rm -f "${file}" || true) - done | \ - scripts/merge-auto-explain-logs | \ - gzip -c > "${RSPEC_AUTO_EXPLAIN_LOG_PATH}" -} diff --git a/spec/support/database/auto_explain.rb b/spec/support/database/auto_explain.rb index 799457034a10e221b338df87ace408551a9c520c..8a83b2819ba02847a6cc651b9031c7d48c9a7bcf 100644 --- a/spec/support/database/auto_explain.rb +++ b/spec/support/database/auto_explain.rb @@ -120,8 +120,8 @@ def record_auto_explain?(connection) return false if connection.database_version.to_s[0..1].to_i < 14 return false if connection.select_one('SHOW is_superuser')['is_superuser'] != 'on' - # This condition matches the pipeline rules for if-merge-request-labels-record-queries - return true if ENV['CI_MERGE_REQUEST_LABELS']&.include?('pipeline:record-queries') + # This condition matches the pipeline rules for if-merge-request + return true if %w[detached merged_result].include?(ENV['CI_MERGE_REQUEST_EVENT_TYPE']) # This condition matches the pipeline rules for if-default-branch-refs ENV['CI_COMMIT_REF_NAME'] == ENV['CI_DEFAULT_BRANCH'] && !ENV['CI_MERGE_REQUEST_IID']