diff --git a/.gitlab/ci/package-and-test/main.gitlab-ci.yml b/.gitlab/ci/package-and-test/main.gitlab-ci.yml index e90c0bbc7c4e30e75df49fa55bbab2505a284d23..3398ba5866414088b429cd95ab5fcac6be572a28 100644 --- a/.gitlab/ci/package-and-test/main.gitlab-ci.yml +++ b/.gitlab/ci/package-and-test/main.gitlab-ci.yml @@ -103,6 +103,19 @@ instance-ff-inverse: # ------------------------------------------ # Jobs with parallel variant # ------------------------------------------ + +# ========== instance =========== +instance: + extends: + - .parallel + - .qa + variables: + QA_SCENARIO: Test::Instance::Image + rules: + - !reference [.rules:test:feature-flags-set, rules] # always run instance to validate ff change + - !reference [.rules:test:qa-parallel, rules] + - if: $QA_SUITES =~ /Test::Instance::All/ + instance-selective: extends: .qa variables: @@ -110,12 +123,29 @@ instance-selective: rules: - !reference [.rules:test:qa-selective, rules] - if: $QA_SUITES =~ /Test::Instance::All/ -instance: + +instance-selective-parallel: extends: - .parallel - - instance-selective + - .qa + variables: + QA_SCENARIO: Test::Instance::Image + rules: + - !reference [.rules:test:qa-selective-parallel, rules] + - if: $QA_SUITES =~ /Test::Instance::All/ + variables: + QA_TESTS: "" + +# ========== praefect =========== + +praefect: + extends: + - .parallel + - .qa + variables: + QA_SCENARIO: Test::Integration::Praefect + QA_CAN_TEST_PRAEFECT: "true" rules: - - !reference [.rules:test:feature-flags-set, rules] # always run instance to validate ff change - !reference [.rules:test:qa-parallel, rules] - if: $QA_SUITES =~ /Test::Instance::All/ @@ -127,10 +157,28 @@ praefect-selective: rules: - !reference [.rules:test:qa-selective, rules] - if: $QA_SUITES =~ /Test::Instance::All/ -praefect: + +praefect-selective-parallel: + extends: + - .qa + - .parallel + variables: + QA_SCENARIO: Test::Integration::Praefect + QA_CAN_TEST_PRAEFECT: "true" + rules: + - !reference [.rules:test:qa-selective-parallel, rules] + - if: $QA_SUITES =~ /Test::Instance::All/ + variables: + QA_TESTS: "" + +# ========== relative-url =========== + +relative-url: extends: + - .qa - .parallel - - praefect-selective + variables: + QA_SCENARIO: Test::Instance::RelativeUrl rules: - !reference [.rules:test:qa-parallel, rules] - if: $QA_SUITES =~ /Test::Instance::All/ @@ -142,10 +190,28 @@ relative-url-selective: rules: - !reference [.rules:test:qa-selective, rules] - if: $QA_SUITES =~ /Test::Instance::All/ -relative-url: + +relative-url-selective-parallel: extends: + - .qa - .parallel - - relative-url-selective + variables: + QA_SCENARIO: Test::Instance::RelativeUrl + rules: + - !reference [.rules:test:qa-selective-parallel, rules] + - if: $QA_SUITES =~ /Test::Instance::All/ + variables: + QA_TESTS: "" + +# ========== decomposition-single-db =========== + +decomposition-single-db: + extends: + - .qa + - .parallel + variables: + QA_SCENARIO: Test::Instance::Image + GITLAB_QA_OPTS: --omnibus-config decomposition_single_db $EXTRA_GITLAB_QA_OPTS rules: - !reference [.rules:test:qa-parallel, rules] - if: $QA_SUITES =~ /Test::Instance::All/ @@ -158,10 +224,30 @@ decomposition-single-db-selective: rules: - !reference [.rules:test:qa-selective, rules] - if: $QA_SUITES =~ /Test::Instance::All/ -decomposition-single-db: + +decomposition-single-db-selective-parallel: + extends: + - .qa + - .parallel + variables: + QA_SCENARIO: Test::Instance::Image + GITLAB_QA_OPTS: --omnibus-config decomposition_single_db $EXTRA_GITLAB_QA_OPTS + rules: + - !reference [.rules:test:qa-selective-parallel, rules] + - if: $QA_SUITES =~ /Test::Instance::All/ + variables: + QA_TESTS: "" + +# ========== decomposition-multiple-db =========== + +decomposition-multiple-db: extends: + - .qa - .parallel - - decomposition-single-db-selective + variables: + QA_SCENARIO: Test::Instance::Image + GITLAB_ALLOW_SEPARATE_CI_DATABASE: "true" + GITLAB_QA_OPTS: --omnibus-config decomposition_multiple_db $EXTRA_GITLAB_QA_OPTS rules: - !reference [.rules:test:qa-parallel, rules] - if: $QA_SUITES =~ /Test::Instance::All/ @@ -175,13 +261,33 @@ decomposition-multiple-db-selective: rules: - !reference [.rules:test:qa-selective, rules] - if: $QA_SUITES =~ /Test::Instance::All/ -decomposition-multiple-db: + +decomposition-multiple-db-selective-parallel: extends: + - .qa - .parallel - - decomposition-multiple-db-selective + variables: + QA_SCENARIO: Test::Instance::Image + GITLAB_ALLOW_SEPARATE_CI_DATABASE: "true" + GITLAB_QA_OPTS: --omnibus-config decomposition_multiple_db $EXTRA_GITLAB_QA_OPTS rules: - - !reference [.rules:test:qa-parallel, rules] + - !reference [.rules:test:qa-selective-parallel, rules] - if: $QA_SUITES =~ /Test::Instance::All/ + variables: + QA_TESTS: "" + +# ========== object-storage =========== + +object-storage: + extends: .qa + parallel: 2 + variables: + QA_SCENARIO: Test::Instance::Image + QA_RSPEC_TAGS: --tag object_storage + GITLAB_QA_OPTS: --omnibus-config object_storage $EXTRA_GITLAB_QA_OPTS + rules: + - !reference [.rules:test:qa-parallel, rules] + - if: $QA_SUITES =~ /Test::Instance::ObjectStorage/ object-storage-selective: extends: .qa @@ -192,12 +298,30 @@ object-storage-selective: rules: - !reference [.rules:test:qa-selective, rules] - if: $QA_SUITES =~ /Test::Instance::ObjectStorage/ -object-storage: - extends: object-storage-selective + +object-storage-selective-parallel: + extends: .qa parallel: 2 + variables: + QA_SCENARIO: Test::Instance::Image + QA_RSPEC_TAGS: --tag object_storage + GITLAB_QA_OPTS: --omnibus-config object_storage $EXTRA_GITLAB_QA_OPTS rules: - - !reference [.rules:test:qa-parallel, rules] + - !reference [.rules:test:qa-selective-parallel, rules] - if: $QA_SUITES =~ /Test::Instance::ObjectStorage/ + variables: + QA_TESTS: "" + +# ========== object-storage-aws =========== + +object-storage-aws: + extends: object-storage + variables: + AWS_S3_ACCESS_KEY: $QA_AWS_S3_ACCESS_KEY + AWS_S3_BUCKET_NAME: $QA_AWS_S3_BUCKET_NAME + AWS_S3_KEY_ID: $QA_AWS_S3_KEY_ID + AWS_S3_REGION: $QA_AWS_S3_REGION + GITLAB_QA_OPTS: --omnibus-config object_storage_aws $EXTRA_GITLAB_QA_OPTS object-storage-aws-selective: extends: object-storage-selective @@ -207,11 +331,27 @@ object-storage-aws-selective: AWS_S3_KEY_ID: $QA_AWS_S3_KEY_ID AWS_S3_REGION: $QA_AWS_S3_REGION GITLAB_QA_OPTS: --omnibus-config object_storage_aws $EXTRA_GITLAB_QA_OPTS -object-storage-aws: - extends: object-storage-aws-selective - parallel: 2 - rules: - - !reference [object-storage, rules] + +object-storage-aws-selective-parallel: + extends: object-storage-selective-parallel + variables: + AWS_S3_ACCESS_KEY: $QA_AWS_S3_ACCESS_KEY + AWS_S3_BUCKET_NAME: $QA_AWS_S3_BUCKET_NAME + AWS_S3_KEY_ID: $QA_AWS_S3_KEY_ID + AWS_S3_REGION: $QA_AWS_S3_REGION + GITLAB_QA_OPTS: --omnibus-config object_storage_aws $EXTRA_GITLAB_QA_OPTS + + +# ========== object-storage-gcs =========== + +object-storage-gcs: + extends: object-storage + variables: + GCS_BUCKET_NAME: $QA_GCS_BUCKET_NAME + GOOGLE_PROJECT: $QA_GOOGLE_PROJECT + GOOGLE_JSON_KEY: $QA_GOOGLE_JSON_KEY + GOOGLE_CLIENT_EMAIL: $QA_GOOGLE_CLIENT_EMAIL + GITLAB_QA_OPTS: --omnibus-config object_storage_gcs $EXTRA_GITLAB_QA_OPTS object-storage-gcs-selective: extends: object-storage-selective @@ -221,11 +361,28 @@ object-storage-gcs-selective: GOOGLE_JSON_KEY: $QA_GOOGLE_JSON_KEY GOOGLE_CLIENT_EMAIL: $QA_GOOGLE_CLIENT_EMAIL GITLAB_QA_OPTS: --omnibus-config object_storage_gcs $EXTRA_GITLAB_QA_OPTS -object-storage-gcs: - extends: object-storage-gcs-selective + +object-storage-gcs-selective-parallel: + extends: object-storage-selective-parallel + variables: + GCS_BUCKET_NAME: $QA_GCS_BUCKET_NAME + GOOGLE_PROJECT: $QA_GOOGLE_PROJECT + GOOGLE_JSON_KEY: $QA_GOOGLE_JSON_KEY + GOOGLE_CLIENT_EMAIL: $QA_GOOGLE_CLIENT_EMAIL + GITLAB_QA_OPTS: --omnibus-config object_storage_gcs $EXTRA_GITLAB_QA_OPTS + +# ========== packages =========== + +packages: + extends: .qa parallel: 2 + variables: + QA_SCENARIO: Test::Instance::Image + QA_RSPEC_TAGS: --tag packages + GITLAB_QA_OPTS: --omnibus-config packages $EXTRA_GITLAB_QA_OPTS rules: - - !reference [object-storage, rules] + - !reference [.rules:test:qa-parallel, rules] + - if: $QA_SUITES =~ /Test::Instance::Packages/ packages-selective: extends: .qa @@ -236,12 +393,19 @@ packages-selective: rules: - !reference [.rules:test:qa-selective, rules] - if: $QA_SUITES =~ /Test::Instance::Packages/ -packages: - extends: packages-selective + +packages-selective-parallel: + extends: .qa parallel: 2 + variables: + QA_SCENARIO: Test::Instance::Image + QA_RSPEC_TAGS: --tag packages + GITLAB_QA_OPTS: --omnibus-config packages $EXTRA_GITLAB_QA_OPTS rules: - - !reference [.rules:test:qa-parallel, rules] + - !reference [.rules:test:qa-selective-parallel, rules] - if: $QA_SUITES =~ /Test::Instance::Packages/ + variables: + QA_TESTS: "" # ------------------------------------------ # Non parallel jobs @@ -474,7 +638,7 @@ elasticsearch: - !reference [.rules:test:manual, rules] registry-object-storage-tls: - extends: object-storage-aws-selective + extends: object-storage-aws-selective-parallel variables: QA_SCENARIO: Test::Integration::RegistryTLS QA_RSPEC_TAGS: "" diff --git a/.gitlab/ci/qa-common/main.gitlab-ci.yml b/.gitlab/ci/qa-common/main.gitlab-ci.yml index 48d8d8ecb2d3cadd3d8826e5e3e743a98df1231b..c557e7c3fdcd6b9dc546103d4f22fa7980c87671 100644 --- a/.gitlab/ci/qa-common/main.gitlab-ci.yml +++ b/.gitlab/ci/qa-common/main.gitlab-ci.yml @@ -163,8 +163,10 @@ stages: KNAPSACK_DIR: ${CI_PROJECT_DIR}/qa/knapsack GIT_STRATEGY: none script: + - echo "KNAPSACK_TEST_FILE_PATTERN is ${KNAPSACK_TEST_FILE_PATTERN}" # when using qa-image, code runs in /home/gitlab/qa folder - bundle exec rake "knapsack:download[test]" + - '[ -n "$QA_TESTS" ] && bundle exec rake "knapsack:create_reports_for_selective"' - mkdir -p "$KNAPSACK_DIR" && cp knapsack/*.json "${KNAPSACK_DIR}/" allow_failure: true artifacts: diff --git a/.gitlab/ci/qa-common/rules.gitlab-ci.yml b/.gitlab/ci/qa-common/rules.gitlab-ci.yml index 3580339921d91c46ba4719c12c097d500d475b50..4d0e0138443999dcf0698b2d94931656882fcfde 100644 --- a/.gitlab/ci/qa-common/rules.gitlab-ci.yml +++ b/.gitlab/ci/qa-common/rules.gitlab-ci.yml @@ -1,5 +1,5 @@ # Specific specs passed -.specific-specs: &specific-specs +.specs-specified: &specs-specified if: $QA_TESTS != "" # No specific specs passed @@ -10,6 +10,14 @@ .feature-flags-set: &feature-flags-set if: $QA_FEATURE_FLAGS =~ /enabled|disabled/ +# Specific specs specified +.spec-file-specified: &spec-file-specified + if: $QA_TESTS =~ /_spec\.rb/ + +# Specs directory specified +.spec-directory-specified: &spec-directory-specified + if: $QA_TESTS != "" && $QA_TESTS !~ /_spec\.rb/ + # Manually trigger job on ff changes but with default ff state instead of inverted .feature-flags-set-manual: &feature-flags-set-manual <<: *feature-flags-set @@ -96,11 +104,25 @@ when: never - <<: *feature-flags-set when: never + - <<: *spec-directory-specified + when: never + +.rules:test:qa-selective-parallel: + rules: + # always run parallel with full suite when framework changes present or ff state changed + - <<: *qa-run-all-tests + when: never + - <<: *all-specs + when: never + - <<: *feature-flags-set + when: never + - <<: *spec-file-specified + when: never .rules:test:qa-parallel: rules: - *qa-run-all-tests - - <<: *specific-specs + - <<: *specs-specified when: manual allow_failure: true variables: diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml index 36e2c3387480f3d77e644412442d009b8fd86218..cf71081a37f043ad35f7432244e5fc853c9a3832 100644 --- a/.gitlab/ci/setup.gitlab-ci.yml +++ b/.gitlab/ci/setup.gitlab-ci.yml @@ -171,3 +171,4 @@ e2e-test-pipeline-generate: expire_in: 1 day paths: - '*-pipeline.yml' + - "${CI_PROJECT_DIR}/qa_tests_vars.env" diff --git a/doc/development/testing_guide/end_to_end/index.md b/doc/development/testing_guide/end_to_end/index.md index 07a5397a80249ed5fef93f5e69bec31fab5dc0de..7932d4131f3dd158f08280d9c5845a2f0249c687 100644 --- a/doc/development/testing_guide/end_to_end/index.md +++ b/doc/development/testing_guide/end_to_end/index.md @@ -165,8 +165,8 @@ In order to limit amount of tests executed in a merge request, dynamic selection on changed files and merge request labels. Following criteria determine which tests will run: 1. Changes in `qa` framework code would execute the full suite -1. Changes in particular `_spec.rb` file in `qa` folder would execute only that particular test -1. Merge request with backend changes and label `devops::manage` would execute all e2e tests related to `manage` stage +1. Changes in particular `_spec.rb` file in `qa` folder would execute only that particular test. In this case knapsack will not be used to run jobs in parallel. +1. Merge request with backend changes and label `devops::manage` would execute all e2e tests related to `manage` stage. Jobs will be run in parallel in this case using knapsack. #### Overriding selective test execution diff --git a/qa/qa/support/knapsack_report.rb b/qa/qa/support/knapsack_report.rb index 27d8b144f3bacf39c5566bf39b272b1f225745a1..d7bf8b769248a75bc1ccc44dcb1ec09991bebc5b 100644 --- a/qa/qa/support/knapsack_report.rb +++ b/qa/qa/support/knapsack_report.rb @@ -46,6 +46,28 @@ def download_report logger.warn("Falling back to '#{FALLBACK_REPORT}'") end + # Create a copy of the report that contains the selective tests and has '-selective' suffix + # + # @param [String] qa_tests + # @return [void] + def create_for_selective(qa_tests) + timed_specs = JSON.parse(File.read(report_path)) + + qa_tests_array = qa_tests.split(' ') + filtered_timed_specs = timed_specs.select { |k, _| qa_tests_array.any? { |qa_test| k.include? qa_test } } + File.write(selective_path, filtered_timed_specs.to_json) + end + + # Add '-selective-parallel' suffix to report name + # + # @return [String] + def selective_path + extension = File.extname(report_path) + directory = File.dirname(report_path) + file_name = File.basename(report_path, extension) + File.join(directory, "#{file_name}-selective-parallel#{extension}") + end + # Rename and move new regenerated report to a separate folder used to indicate report name # # @return [void] diff --git a/qa/spec/fixtures/knapsack_report/instance-selective-parallel.json b/qa/spec/fixtures/knapsack_report/instance-selective-parallel.json new file mode 100644 index 0000000000000000000000000000000000000000..adf506c9d3072a1cea8f18f447317187aac89dbe --- /dev/null +++ b/qa/spec/fixtures/knapsack_report/instance-selective-parallel.json @@ -0,0 +1,5 @@ +{ + "qa/specs/features/ee/browser_ui/3_create/repository/code_owners_with_protected_branch_and_squashed_commits_spec.rb": 18.85673829499956, + "qa/specs/features/api/3_create/repository/files_spec.rb": 3.180753622999873, + "qa/specs/features/browser_ui/3_create/web_ide_old/server_hooks_custom_error_message_spec.rb": 0.010764157999801682 +} diff --git a/qa/spec/fixtures/knapsack_report/instance.json b/qa/spec/fixtures/knapsack_report/instance.json new file mode 100644 index 0000000000000000000000000000000000000000..3d659bc53fbd409cbfaf4497e6c5d90bf6605938 --- /dev/null +++ b/qa/spec/fixtures/knapsack_report/instance.json @@ -0,0 +1,7 @@ +{ + "qa/specs/features/ee/browser_ui/3_create/repository/code_owners_with_protected_branch_and_squashed_commits_spec.rb": 18.85673829499956, + "qa/specs/features/api/3_create/repository/files_spec.rb": 3.180753622999873, + "qa/specs/features/browser_ui/3_create/web_ide_old/server_hooks_custom_error_message_spec.rb": 0.010764157999801682, + "qa/specs/features/api/9_data_stores/users_spec.rb": 0.2808277129997805, + "qa/specs/features/api/2_plan/closes_issue_via_pushing_a_commit_spec.rb": 4.882168451000325 +} diff --git a/qa/spec/support/knapsack_report_spec.rb b/qa/spec/support/knapsack_report_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..914a30513e5e2b0ecc9ae962fb81792453abe3dc --- /dev/null +++ b/qa/spec/support/knapsack_report_spec.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +RSpec.describe QA::Support::KnapsackReport do + subject(:knapsack_report) { described_class.new('instance') } + + describe '#create_for_selective' do + let(:qa_tests) do + <<~CMD + qa/specs/features/api/3_create + qa/specs/features/browser_ui/3_create/ + qa/specs/features/ee/api/3_create/ + qa/specs/features/ee/browser_ui/3_create/ + CMD + end + + let(:fixtures_path) { 'spec/fixtures/knapsack_report' } + let(:expected_output) { JSON.parse(File.read(File.join(fixtures_path, 'instance-selective-parallel.json'))) } + + before do + allow(File).to receive(:read).and_call_original + allow(File).to receive(:read) + .with('knapsack/instance.json') + .and_return(File.read(File.join(fixtures_path, 'instance.json'))) + end + + it 'creates a filtered file based on qa_tests' do + expect(File).to receive(:write) + .with('knapsack/instance-selective-parallel.json', expected_output.to_json) + + knapsack_report.create_for_selective(qa_tests) + end + end + + describe '#selective_path' do + it 'returns the path with file name suffixed with -selective-parallel' do + expect(knapsack_report.selective_path).to eq('knapsack/instance-selective-parallel.json') + end + end +end diff --git a/qa/tasks/ci.rake b/qa/tasks/ci.rake index e5f4acb158bfae585d5ca07e50cef07d4777a8c8..3dfad6a82fdabb6fece013487a61656e81cfa0e1 100644 --- a/qa/tasks/ci.rake +++ b/qa/tasks/ci.rake @@ -30,6 +30,18 @@ namespace :ci do # on run-all label of framework changes do not infer specific tests tests = run_all_label_present || qa_changes.framework_changes? ? nil : qa_changes.qa_tests + # When QA_TESTS only contain folders and no specific specs, populate KNAPSACK_TEST_FILE_PATTERN + if tests && tests.split(' ').none? { |item| item.include?('_spec') } + test_paths = tests.split(' ').map { |item| "#{item}**/*" } + + files_pattern = "{#{test_paths.join(',')}}" + + logger.info(" Files pattern for tests: #{files_pattern}") + append_to_file(env_file, <<~TXT) + KNAPSACK_TEST_FILE_PATTERN='#{files_pattern}' + TXT + end + if run_all_label_present logger.info(" merge request has pipeline:run-all-e2e label, full test suite will be executed") append_to_file(env_file, "QA_RUN_ALL_E2E_LABEL=true\n") diff --git a/qa/tasks/knapsack.rake b/qa/tasks/knapsack.rake index 5e60703ced369b04a6fb00f1001d9b51c0dfdd4e..b8a8d6e114567ebe151272725496cd069e5b87fa 100644 --- a/qa/tasks/knapsack.rake +++ b/qa/tasks/knapsack.rake @@ -42,6 +42,19 @@ namespace :knapsack do end end + desc "Create knapsack reports from existing reports for selective jobs" + task :create_reports_for_selective do + reports = Dir.glob("knapsack/*").map { |file| file.match(%r{.*/(.*)?\.json})[1] } + + reports.each do |report_name| + unless report_name.include?('-selective-parallel') + QA::Support::KnapsackReport.new(report_name).create_for_selective(ENV['QA_TESTS']) + end + rescue StandardError => e + QA::Runtime::Logger.error(e) + end + end + desc "Merge and upload knapsack report" task :upload, [:glob] do |_task, args| QA::Support::KnapsackReport.configure! diff --git a/scripts/generate-e2e-pipeline b/scripts/generate-e2e-pipeline index 3f30fb86ccc36381267dab60df5776feb82cff56..b0b2b0b0035bbae4eb0ebd40c6d8d59acfe3c075 100755 --- a/scripts/generate-e2e-pipeline +++ b/scripts/generate-e2e-pipeline @@ -49,6 +49,7 @@ variables: QA_SAVE_TEST_METRICS: "${QA_SAVE_TEST_METRICS:-false}" QA_SUITES: "$QA_SUITES" QA_TESTS: "$QA_TESTS" + KNAPSACK_TEST_FILE_PATTERN: "$KNAPSACK_TEST_FILE_PATTERN" YML ) @@ -63,3 +64,4 @@ for key in "${!qa_pipelines[@]}"; do done echoinfo "Successfully generated qa pipeline files" +echo "$variables"