diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index f3eae6371b19a21559f5f3a0098187aaeb128f93..5ae218865fd9e1bfaccce73d93faafdfdc48ffaf 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -66,6 +66,8 @@ - scripts/gitaly-test-spawn - date - 'export KNAPSACK_TEST_FILE_PATTERN=$(ruby -r./lib/quality/test_level.rb -e "puts Quality::TestLevel.new.pattern(:${TEST_LEVEL})")' + - mkdir -p tmp/memory_test + - export MEMORY_TEST_PATH="tmp/memory_test/${TEST_TOOL}_${TEST_LEVEL}_${DATABASE}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_memory.csv" - knapsack rspec "--color --format documentation --format RspecJunitFormatter --out junit_rspec.xml --tag level:${TEST_LEVEL} --tag ~geo" - date artifacts: @@ -77,6 +79,7 @@ - rspec_flaky/ - rspec_profiling/ - tmp/capybara/ + - tmp/memory_test/ # reports: # junit: junit_rspec.xml @@ -273,6 +276,7 @@ coverage: stage: post-test script: - bundle exec scripts/merge-simplecov + - bundle exec scripts/gather-test-memory-data coverage: '/LOC \((\d+\.\d+%)\) covered.$/' artifacts: name: coverage @@ -280,6 +284,7 @@ coverage: paths: - coverage/index.html - coverage/assets/ + - tmp/memory_test/ except: - /(^docs[\/-].*|.*-docs$)/ - /(^qa[\/-].*|.*-qa$)/ @@ -307,6 +312,8 @@ coverage: - scripts/gitaly-test-spawn - date - 'export KNAPSACK_TEST_FILE_PATTERN=$(ruby -r./lib/quality/test_level.rb -e "puts Quality::TestLevel.new(%(ee/)).pattern(:${TEST_LEVEL})")' + - mkdir -p tmp/memory_test + - export MEMORY_TEST_PATH="tmp/memory_test/ee_${TEST_TOOL}_${TEST_LEVEL}_${DATABASE}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_memory.csv" - knapsack rspec "--color --format documentation --format RspecJunitFormatter --out junit_rspec.xml --tag level:${TEST_LEVEL} --tag ~geo" - date @@ -332,11 +339,15 @@ rspec system pg ee: script: - JOB_NAME=( $CI_JOB_NAME ) - TEST_TOOL=${JOB_NAME[0]} + - TEST_LEVEL=${JOB_NAME[1]} + - DATABASE=${JOB_NAME[2]} - export KNAPSACK_TEST_FILE_PATTERN="ee/spec/**{,/*/**}/*_spec.rb" KNAPSACK_GENERATE_REPORT=true CACHE_CLASSES=true - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${TEST_TOOL}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json - cp ${EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH} - source scripts/prepare_postgres_fdw.sh - scripts/gitaly-test-spawn + - mkdir -p tmp/memory_test + - export MEMORY_TEST_PATH="tmp/memory_test/ee_${TEST_TOOL}_${TEST_LEVEL}_${DATABASE}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_memory.csv" - knapsack rspec "-Ispec --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml --tag geo" rspec geo pg ee: diff --git a/scripts/gather-test-memory-data b/scripts/gather-test-memory-data new file mode 100755 index 0000000000000000000000000000000000000000..9992a83e6a61f36754e9eca4328082f03cd51040 --- /dev/null +++ b/scripts/gather-test-memory-data @@ -0,0 +1,21 @@ +#!/usr/bin/env ruby + +require 'csv' + +def join_csv_files(output_path, input_paths) + return if input_paths.empty? + + input_csvs = input_paths.map do |input_path| + CSV.read(input_path, headers: true) + end + + CSV.open(output_path, "w", headers: input_csvs.first.headers, write_headers: true) do |output_csv| + input_csvs.each do |input_csv| + input_csv.each do |line| + output_csv << line + end + end + end +end + +join_csv_files('tmp/memory_test/report.csv', Dir['tmp/memory_test/*.csv'].sort) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 859b08e9f5fdcc4a3604d2e06d048fb6d980aeed..cc7b07de9f0dfa00ca3507c72e7eda53460e3320 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -105,6 +105,7 @@ config.include RedisHelpers config.include Rails.application.routes.url_helpers, type: :routing config.include PolicyHelpers, type: :policy + config.include MemoryUsageHelper if ENV['CI'] # This includes the first try, i.e. tests will be run 4 times before failing. diff --git a/spec/support/helpers/memory_usage_helper.rb b/spec/support/helpers/memory_usage_helper.rb new file mode 100644 index 0000000000000000000000000000000000000000..984ea8cc571456e67e335a0088ee2d68d211afa7 --- /dev/null +++ b/spec/support/helpers/memory_usage_helper.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module MemoryUsageHelper + extend ActiveSupport::Concern + + def gather_memory_data(csv_path) + write_csv_entry(csv_path, + { + example_group_path: TestEnv.topmost_example_group[:location], + example_group_description: TestEnv.topmost_example_group[:description], + time: Time.current, + job_name: ENV['CI_JOB_NAME'] + }.merge(get_memory_usage)) + end + + def write_csv_entry(path, entry) + CSV.open(path, "a", headers: entry.keys, write_headers: !File.exist?(path)) do |file| + file << entry.values + end + end + + def get_memory_usage + output, status = Gitlab::Popen.popen(%w(free -m)) + abort "`free -m` return code is #{status}: #{output}" unless status.zero? + + result = output.split("\n")[1].split(" ")[1..-1] + attrs = %i(m_total m_used m_free m_shared m_buffers_cache m_available).freeze + + attrs.zip(result).to_h + end + + included do |config| + config.after(:all) do + gather_memory_data(ENV['MEMORY_TEST_PATH']) if ENV['MEMORY_TEST_PATH'] + end + end +end diff --git a/spec/support/helpers/test_env.rb b/spec/support/helpers/test_env.rb index cae525af784361486c533ae46bc352691b41b730..b3a3387338892b59bd5d8f2d498f6b43e2a9b8a8 100644 --- a/spec/support/helpers/test_env.rb +++ b/spec/support/helpers/test_env.rb @@ -2,6 +2,7 @@ require 'toml-rb' module TestEnv + extend ActiveSupport::Concern extend self ComponentFailedToInstallError = Class.new(StandardError) @@ -108,6 +109,12 @@ def init(opts = {}) setup_forked_repo end + included do |config| + config.append_before do + set_current_example_group + end + end + def disable_mailer allow_any_instance_of(NotificationService).to receive(:mailer) .and_return(double.as_null_object) @@ -297,8 +304,23 @@ def with_empty_bare_repository(name = nil) FileUtils.rm_rf(path) end + def current_example_group + Thread.current[:current_example_group] + end + + # looking for a top-level `describe` + def topmost_example_group + example_group = current_example_group + example_group = example_group[:parent_example_group] until example_group[:parent_example_group].nil? + example_group + end + private + def set_current_example_group + Thread.current[:current_example_group] = ::RSpec.current_example.metadata[:example_group] + end + # These are directories that should be preserved at cleanup time def test_dirs @test_dirs ||= %w[