diff --git a/.gitlab/ci/build-images.gitlab-ci.yml b/.gitlab/ci/build-images.gitlab-ci.yml index 1e228fdafde365fa2d4a54c283216ba02b951ae3..1621d474d39f1198f62704f85095d64403a778fb 100644 --- a/.gitlab/ci/build-images.gitlab-ci.yml +++ b/.gitlab/ci/build-images.gitlab-ci.yml @@ -29,6 +29,13 @@ build-qa-image as-if-foss: - .as-if-foss - .build-images:rules:build-qa-image-as-if-foss +follow-up:build-qa-image: + extends: + - build-qa-image + - .qa:rules:follow-up-e2e + needs: + - manual:e2e-test-pipeline-generate + retag-gdk-image: extends: - .base-image-build @@ -82,3 +89,9 @@ build-assets-image as-if-foss: - .as-if-foss - .build-images:rules:build-assets-image-as-if-foss needs: ["compile-production-assets as-if-foss"] + +follow-up:build-assets-image: + extends: + - build-assets-image + - .qa:rules:follow-up-e2e + needs: ["follow-up:compile-production-assets"] diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index b1b4079b3e4d60335f7b7ccfe60ede8e151ee448..d46bbed3db8a5045f42c7e4227971dd9aede1cb6 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -58,6 +58,13 @@ compile-production-assets as-if-foss: - .as-if-foss - .frontend:rules:compile-production-assets-as-if-foss +follow-up:compile-production-assets: + extends: + - compile-production-assets + - .qa:rules:follow-up-e2e + needs: + - manual:e2e-test-pipeline-generate + compile-test-assets: extends: - .compile-assets-base diff --git a/.gitlab/ci/qa.gitlab-ci.yml b/.gitlab/ci/qa.gitlab-ci.yml index 5acdf5ab021ce94120a968e0db56d8f45b3c3147..917b875bc9b7df007685f83413cfaa76fc94544c 100644 --- a/.gitlab/ci/qa.gitlab-ci.yml +++ b/.gitlab/ci/qa.gitlab-ci.yml @@ -101,11 +101,13 @@ trigger-omnibus: branch: $TRIGGER_BRANCH strategy: depend -# Same as trigger-omnibus but is manual and runs follow-up-e2e:package-and-test-ee automatically right after -trigger-omnibus-and-follow-up-e2e: +follow-up:trigger-omnibus: extends: - trigger-omnibus - - .qa:rules:manual-omnibus-and-follow-up-e2e + - .qa:rules:follow-up-e2e + needs: + - follow-up:trigger-omnibus-env + - follow-up:build-assets-image trigger-omnibus as-if-foss: extends: @@ -132,19 +134,15 @@ e2e:package-and-test-ee: QA_RUN_TYPE: e2e-package-and-test PIPELINE_NAME: E2E Omnibus GitLab EE -# Same as e2e:package-and-test-ee but runs automatically after trigger-omnibus-and-follow-up-e2e -follow-up-e2e:package-and-test-ee: +# Same as e2e:package-and-test-ee but runs automatically after manual:e2e-test-pipeline-generate +follow-up:e2e:package-and-test-ee: extends: - - .e2e-trigger-base + - e2e:package-and-test-ee - .qa:rules:follow-up-e2e needs: - - build-qa-image - - trigger-omnibus-and-follow-up-e2e - - e2e-test-pipeline-generate - variables: - RELEASE: "${REGISTRY_HOST}/${REGISTRY_GROUP}/build/omnibus-gitlab-mirror/gitlab-ee:${RELEASE_IMAGE_TAG}" - QA_RUN_TYPE: e2e-package-and-test - PIPELINE_NAME: E2E Omnibus GitLab EE + - follow-up:build-qa-image + - follow-up:trigger-omnibus + - manual:e2e-test-pipeline-generate e2e:post-run-e2e-message: extends: diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index 2944dd8afc853f475403ed32da2e2a16dffee9ce..33830267a5f1d6f52ffe8e9bc3683dea7b3ed2c7 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -155,12 +155,6 @@ .if-dot-com-gitlab-org-merge-request: &if-dot-com-gitlab-org-merge-request if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" && ($CI_MERGE_REQUEST_EVENT_TYPE == "merged_result" || $CI_MERGE_REQUEST_EVENT_TYPE == "detached")' -.if-dot-com-gitlab-org-and-security-merge-request: &if-dot-com-gitlab-org-and-security-merge-request - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org/security" && ($CI_MERGE_REQUEST_EVENT_TYPE == "merged_result" || $CI_MERGE_REQUEST_EVENT_TYPE == "detached")' - -.if-dot-com-gitlab-org-and-security-tag: &if-dot-com-gitlab-org-and-security-tag - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org/security" && $CI_COMMIT_TAG' - .if-ruby-branch: &if-ruby-branch if: '$CI_COMMIT_BRANCH =~ /^ruby\d+(_\d)*$/ || $CI_MERGE_REQUEST_LABELS =~ /pipeline:run-in-ruby\d+(_\d)*/' @@ -876,33 +870,37 @@ ###################### .build-images:rules:build-qa-image-merge-requests: rules: - - <<: *if-not-canonical-namespace - when: never - - <<: *if-merge-request-labels-pipeline-expedite - when: never - if: '$ENABLE_BUILD_QA_IMAGE == "true"' + - <<: *if-merge-request-labels-run-all-e2e + - <<: *if-merge-request-labels-run-review-app + - <<: *if-merge-request + changes: *qa-patterns - <<: *if-merge-request-targeting-stable-branch changes: *setup-test-env-patterns - - <<: *if-merge-request-labels-run-review-app - <<: *if-merge-request changes: *ci-build-images-patterns + - <<: *if-merge-request-not-approved + when: never - <<: *if-merge-request changes: *code-qa-patterns # Rules to support .qa:rules:package-and-test-ee - <<: *if-merge-request changes: *dependency-patterns - - <<: *if-merge-request-labels-run-all-e2e - <<: *if-merge-request changes: *feature-flag-development-config-patterns - <<: *if-merge-request changes: *nodejs-patterns - <<: *if-merge-request changes: *ci-qa-patterns - - <<: *if-force-ci .build-images:rules:build-qa-image: rules: - - !reference [".build-images:rules:build-qa-image-merge-requests", "rules"] + - <<: *if-not-canonical-namespace + when: never + - <<: *if-merge-request-labels-pipeline-expedite + when: never + - <<: *if-ruby-branch + - <<: *if-force-ci - <<: *if-auto-deploy-branches variables: ARCH: amd64,arm64 @@ -917,16 +915,20 @@ - <<: *if-dot-com-gitlab-org-schedule variables: ARCH: amd64,arm64 - - <<: *if-ruby-branch + - !reference [".build-images:rules:build-qa-image-merge-requests", rules] - !reference [".releases:rules:canonical-dot-com-gitlab-stable-branch-only-setup-test-env", rules] .build-images:rules:build-qa-image-as-if-foss: rules: + - <<: *if-not-canonical-namespace + when: never + - <<: *if-merge-request-labels-pipeline-expedite + when: never - !reference [".strict-ee-only-rules", rules] - - !reference [".build-images:rules:build-qa-image-merge-requests", "rules"] - <<: *if-default-branch-schedule-nightly variables: ARCH: amd64,arm64 + - !reference [".build-images:rules:build-qa-image-merge-requests", rules] # When new minor release tag is pushed, re-tag gdk image with pattern ignored by registry cleanup .build-images:rules:retag-gdk-image: @@ -968,7 +970,7 @@ changes: *nodejs-patterns - <<: *if-merge-request changes: *ci-qa-patterns - - <<: *if-dot-com-gitlab-org-and-security-merge-request + - <<: *if-security-merge-request changes: *code-patterns - <<: *if-merge-request changes: *code-qa-patterns @@ -1194,19 +1196,19 @@ - <<: *if-auto-deploy-branches - <<: *if-ruby-branch - <<: *if-force-ci - # Rules to support .qa:rules:package-and-test - - <<: *if-default-branch-schedule-nightly - <<: *if-merge-request-labels-run-all-e2e - <<: *if-merge-request-labels-run-review-app + - <<: *if-merge-request + changes: *qa-patterns - <<: *if-merge-request-targeting-stable-branch changes: *setup-test-env-patterns - <<: *if-default-refs changes: *ci-build-images-patterns - <<: *if-default-refs changes: *workhorse-patterns - # TODO: Below are broad so we only want to run if it's approved if it's MR - # - <<: *if-merge-request-not-approved - # when: never + - <<: *if-merge-request-not-approved + when: never + # Rules to support .qa:rules:package-and-test - <<: *if-merge-request changes: *dependency-patterns - <<: *if-default-refs @@ -1490,6 +1492,10 @@ when: never - <<: *if-merge-request-labels-pipeline-expedite when: never + - <<: *if-ruby-branch + allow_failure: true + - <<: *if-merge-request-labels-run-all-e2e + allow_failure: true - <<: *if-merge-request-targeting-stable-branch changes: *setup-test-env-patterns allow_failure: true @@ -1501,13 +1507,10 @@ allow_failure: true - <<: *if-dot-com-gitlab-org-schedule allow_failure: true - - <<: *if-ruby-branch # Rules to support .qa:rules:package-and-test-ee - <<: *if-merge-request changes: *dependency-patterns allow_failure: true - - <<: *if-merge-request-labels-run-all-e2e - allow_failure: true - <<: *if-merge-request changes: *feature-flag-development-config-patterns allow_failure: true @@ -1525,16 +1528,18 @@ rules: # From .qa:rules:package-and-test-common - !reference [".qa:rules:package-and-test-never-run", rules] - - <<: *if-merge-request-targeting-stable-branch - changes: *setup-test-env-patterns - <<: *if-ruby-branch + - <<: *if-merge-request-labels-run-all-e2e - <<: *if-merge-request changes: *qa-patterns + - <<: *if-merge-request-targeting-stable-branch + changes: *setup-test-env-patterns + - <<: *if-merge-request-not-approved + when: never - <<: *if-merge-request changes: *dependency-patterns variables: OMNIBUS_GITLAB_BUILD_ON_ALL_OS: 'true' - - <<: *if-merge-request-labels-run-all-e2e - <<: *if-merge-request changes: *feature-flag-development-config-patterns - <<: *if-merge-request @@ -1543,11 +1548,11 @@ changes: *nodejs-patterns - <<: *if-merge-request changes: *ci-qa-patterns - - <<: *if-dot-com-gitlab-org-and-security-merge-request + - <<: *if-security-merge-request changes: *code-patterns - - <<: *if-force-ci - <<: *if-merge-request-and-specific-devops-stage changes: *code-patterns + - <<: *if-force-ci # From .qa:rules:package-and-test-schedule - <<: *if-dot-com-gitlab-org-schedule - !reference [".qa:rules:code-merge-request", rules] @@ -1564,15 +1569,17 @@ .qa:rules:package-and-test-common: rules: - !reference [".qa:rules:package-and-test-never-run", rules] - - <<: *if-merge-request-targeting-stable-branch - changes: *setup-test-env-patterns - allow_failure: true - <<: *if-ruby-branch allow_failure: true + - <<: *if-merge-request-labels-run-all-e2e + allow_failure: true # Run tests automatically for MRs that touch QA files - <<: *if-merge-request changes: *qa-patterns allow_failure: true + - <<: *if-merge-request-targeting-stable-branch + changes: *setup-test-env-patterns + allow_failure: true # Otherwise, only run tests after the MR is approved - <<: *if-merge-request-not-approved when: never @@ -1580,8 +1587,6 @@ changes: *dependency-patterns variables: OMNIBUS_GITLAB_BUILD_ON_ALL_OS: 'true' - - <<: *if-merge-request-labels-run-all-e2e - allow_failure: true - <<: *if-merge-request changes: *feature-flag-development-config-patterns allow_failure: true @@ -1594,15 +1599,15 @@ - <<: *if-merge-request changes: *ci-qa-patterns allow_failure: true - - <<: *if-dot-com-gitlab-org-and-security-merge-request + - <<: *if-security-merge-request changes: *code-patterns allow_failure: true - - <<: *if-force-ci - when: manual - allow_failure: true - <<: *if-merge-request-and-specific-devops-stage changes: *code-patterns allow_failure: true + - <<: *if-force-ci + when: manual + allow_failure: true # We used to have a rule at the end here that would catch any remaining code MRs and allow the job to be run # manually. That rule is now in ".qa:rules:code-merge-request-manual" so it can be included when needed and we can # still use ".qa:rules:package-and-test-common" in jobs we don't want to be manual. @@ -1673,10 +1678,8 @@ changes: *code-patterns # Note: If any changes are made to this rule, the following should also be updated: -# 1) .qa:rules:manual-omnibus-and-follow-up-e2e -# 2) .qa:rules:follow-up-e2e -# 3) .qa:rules:trigger-omnibus-env -# 4) .qa:rules:post-run-e2e-message +# 1) .qa:rules:trigger-omnibus-env +# 2) .qa:rules:post-run-e2e-message .qa:rules:package-and-test-ee: rules: - !reference [".qa:rules:package-and-test-common", rules] @@ -1686,12 +1689,14 @@ .qa:rules:package-and-test-ce: rules: - !reference [".qa:rules:package-and-test-never-run", rules] - - <<: *if-dot-com-gitlab-org-and-security-merge-request - changes: *ci-build-images-patterns + - <<: *if-security-merge-request + changes: *qa-patterns when: manual allow_failure: true - - <<: *if-dot-com-gitlab-org-and-security-merge-request - changes: *code-qa-patterns + - <<: *if-merge-request-not-approved + when: never + - <<: *if-security-merge-request + changes: *code-patterns when: manual allow_failure: true - <<: *if-default-branch-schedule-nightly @@ -1740,18 +1745,18 @@ - !reference [".qa:rules:package-and-test-nightly", rules] - !reference [".qa:rules:package-and-test-ce", rules] -# These are based on `.qa:rules:trigger-omnibus` except: -# 1. Automatic jobs changed to `when: never.` -# 2. Do not run for specific devops stage -# If any changes are made to this rule, `.qa:rules:follow-up-e2e` should also be updated. -.qa:rules:manual-omnibus-and-follow-up-e2e: +# These provide a manual way to trigger follow-up:e2e:package-and-test-ee +# It is fine if they're overlapping with the automatic ones. +# It'll not hurt and it can simplify the rules, decoupling them. +# If any changes are made to this rule, `.qa:rules:manual-e2e` should also be updated. +.qa:rules:manual-e2e: rules: - !reference [".qa:rules:package-and-test-never-run", rules] - <<: *if-merge-request-and-specific-devops-stage when: never - !reference [".qa:rules:code-merge-request-manual", rules] -# These are based on `.qa:rules:manual-omnibus-and-follow-up-e2e` but with manual jobs changed to automatic +# These are based on `.qa:rules:manual-e2e` but with manual jobs changed to automatic .qa:rules:follow-up-e2e: rules: - !reference [".qa:rules:package-and-test-never-run", rules] @@ -1764,16 +1769,21 @@ rules: # From .qa:rules:package-and-test-common - !reference [".qa:rules:package-and-test-never-run", rules] + - <<: *if-ruby-branch + when: never + - <<: *if-merge-request-labels-run-all-e2e + when: never + - <<: *if-merge-request + changes: *qa-patterns + when: never - <<: *if-merge-request-targeting-stable-branch changes: *setup-test-env-patterns when: never - - <<: *if-ruby-branch + - <<: *if-merge-request-not-approved when: never - <<: *if-merge-request changes: *dependency-patterns when: never - - <<: *if-merge-request-labels-run-all-e2e - when: never - <<: *if-merge-request changes: *feature-flag-development-config-patterns when: never @@ -1786,22 +1796,16 @@ - <<: *if-merge-request changes: *ci-qa-patterns when: never - - <<: *if-merge-request - changes: *qa-patterns - when: never - - <<: *if-dot-com-gitlab-org-and-security-merge-request + - <<: *if-security-merge-request changes: *code-patterns when: never - - <<: *if-force-ci - when: never - <<: *if-merge-request-and-specific-devops-stage when: never + - <<: *if-force-ci + when: never # From .qa:rules:package-and-test-schedule - <<: *if-dot-com-gitlab-org-schedule when: never - # Do not run on unapproved MR - - <<: *if-merge-request-not-approved - when: never - !reference [".qa:rules:code-merge-request-allowed-to-fail", rules] .qa:rules:fulfillment-e2e-quarantine-report: diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml index 7b06ccd5888a917298c5a047708df74816b86805..5818ac3d63b6bdc9fafdc8c3424e813d21b7e049 100644 --- a/.gitlab/ci/setup.gitlab-ci.yml +++ b/.gitlab/ci/setup.gitlab-ci.yml @@ -220,6 +220,16 @@ e2e-test-pipeline-generate: - '*-pipeline.yml' - "${CI_PROJECT_DIR}/qa_tests_vars.env" +# This is the manual start of the chain of E2E jobs +# After playing this, it'll follow up with: +# follow-up:compile-production-assets -> follow-up:build-assets-image --\ +# \-> follow-up:trigger-omnibus-env -> follow-up:trigger-omnibus -> follow-up:e2e:package-and-test-ee +# follow-up:build-qa-image --------------------------------------------------------------------------/ +manual:e2e-test-pipeline-generate: + extends: + - e2e-test-pipeline-generate + - .qa:rules:manual-e2e + trigger-omnibus-env: stage: prepare extends: @@ -263,3 +273,9 @@ trigger-omnibus-env as-if-foss: - compile-production-assets as-if-foss variables: FOSS_ONLY: "1" + +follow-up:trigger-omnibus-env: + extends: + - trigger-omnibus-env + - .qa:rules:follow-up-e2e + needs: ["follow-up:compile-production-assets"] diff --git a/danger/qa_selector/Dangerfile b/danger/qa_selector/Dangerfile index 98417e79782a6f19aa0e25db4353d60078de3ad7..39e0de206634e7a108de34203608961a7808c16c 100644 --- a/danger/qa_selector/Dangerfile +++ b/danger/qa_selector/Dangerfile @@ -50,7 +50,7 @@ if lines_with_testids.any? markdown(<<~MARKDOWN) If the `e2e:package-and-test` job in the `qa` stage has run automatically, please ensure the tests are passing. - If the job has not run, please start the `trigger-omnibus-and-follow-up-e2e` job in the `qa` stage and ensure the tests in `follow-up-e2e:package-and-test-ee` pipeline are passing. + If the job has not run, please start the `manual:e2e-test-pipeline-generate` job in the `prepare` stage and ensure the tests in `follow-up:e2e:package-and-test-ee` pipeline are passing. For the list of known failures please refer to [the latest pipeline triage issue](https://gitlab.com/gitlab-org/quality/pipeline-triage/-/issues). diff --git a/doc/development/pipelines/internals.md b/doc/development/pipelines/internals.md index 20f14e2b19a138366dcd6e8483cafd42c3e26869..055b6841390b0c27ede1cb0618507a354ad2be6b 100644 --- a/doc/development/pipelines/internals.md +++ b/doc/development/pipelines/internals.md @@ -227,9 +227,6 @@ and included in `rules` definitions via [YAML anchors](../../ci/yaml/yaml_optimi | `if-dot-com-gitlab-org-schedule` | Limits jobs creation to scheduled pipelines for the `gitlab-org` group on GitLab.com. | | | `if-dot-com-gitlab-org-master` | Limits jobs creation to the `master` or `main` branch for the `gitlab-org` group on GitLab.com. | | | `if-dot-com-gitlab-org-merge-request` | Limits jobs creation to merge requests for the `gitlab-org` group on GitLab.com. | | -| `if-dot-com-gitlab-org-and-security-tag` | Limits job creation to tags for the `gitlab-org` and `gitlab-org/security` groups on GitLab.com. | | -| `if-dot-com-gitlab-org-and-security-merge-request` | Limit jobs creation to merge requests for the `gitlab-org` and `gitlab-org/security` groups on GitLab.com. | | -| `if-dot-com-gitlab-org-and-security-tag` | Limit jobs creation to tags for the `gitlab-org` and `gitlab-org/security` groups on GitLab.com. | | | `if-dot-com-ee-schedule` | Limits jobs to scheduled pipelines for the `gitlab-org/gitlab` project on GitLab.com. | | <!-- vale gitlab.Substitutions = YES --> diff --git a/scripts/generate-message-to-run-e2e-pipeline.rb b/scripts/generate-message-to-run-e2e-pipeline.rb index 773aa4145d13c91343d4aaaa5e4f00dd6969dc8c..36a1ecc007479c71c4768771b58cb145537c539c 100755 --- a/scripts/generate-message-to-run-e2e-pipeline.rb +++ b/scripts/generate-message-to-run-e2e-pipeline.rb @@ -72,7 +72,7 @@ def content <!-- Run e2e warning begin --> @#{author_username} Some end-to-end (E2E) tests should run based on the stage label. - Please start the `trigger-omnibus-and-follow-up-e2e` job in the `qa` stage and wait for the tests in the `follow-up-e2e:package-and-test-ee` pipeline + Please start the `manual:e2e-test-pipeline-generate` job in the `prepare` stage and wait for the tests in the `follow-up:e2e:package-and-test-ee` pipeline to pass **before merging this MR**. Do not use **Auto-merge**, unless these tests have already completed successfully, because a failure in these tests do not block the auto-merge. (E2E tests are computationally intensive and don't run automatically for every push/rebase, so we ask you to run this job manually at least once.) diff --git a/spec/dot_gitlab_ci/rules_spec.rb b/spec/dot_gitlab_ci/rules_spec.rb index b842ccc318a04bc0fbe5cc209460d283d3b5a56c..3ddf4189f9f0b3b6a462da2e0c6df942d0484bda 100644 --- a/spec/dot_gitlab_ci/rules_spec.rb +++ b/spec/dot_gitlab_ci/rules_spec.rb @@ -26,8 +26,8 @@ end end - describe '.qa:rules:manual-omnibus-and-follow-up-e2e' do - let(:base_rules) { config.dig('.qa:rules:manual-omnibus-and-follow-up-e2e', 'rules') } + describe '.qa:rules:manual-e2e' do + let(:base_rules) { config.dig('.qa:rules:manual-e2e', 'rules') } context 'with .qa:rules:follow-up-e2e' do let(:derived_rules) { config.dig('.qa:rules:follow-up-e2e', 'rules') } diff --git a/spec/scripts/generate_message_to_run_e2e_pipeline_spec.rb b/spec/scripts/generate_message_to_run_e2e_pipeline_spec.rb index 9b19121573984bd08e39719174c670eacec82e3c..54c0a40e39f58e8fe1c34b96bf07f3237ffc2245 100644 --- a/spec/scripts/generate_message_to_run_e2e_pipeline_spec.rb +++ b/spec/scripts/generate_message_to_run_e2e_pipeline_spec.rb @@ -224,7 +224,7 @@ <!-- Run e2e warning begin --> @#{author_username} Some end-to-end (E2E) tests should run based on the stage label. - Please start the `trigger-omnibus-and-follow-up-e2e` job in the `qa` stage and wait for the tests in the `follow-up-e2e:package-and-test-ee` pipeline + Please start the `manual:e2e-test-pipeline-generate` job in the `prepare` stage and wait for the tests in the `follow-up:e2e:package-and-test-ee` pipeline to pass **before merging this MR**. Do not use **Auto-merge**, unless these tests have already completed successfully, because a failure in these tests do not block the auto-merge. (E2E tests are computationally intensive and don't run automatically for every push/rebase, so we ask you to run this job manually at least once.) diff --git a/spec/tooling/danger/stable_branch_spec.rb b/spec/tooling/danger/stable_branch_spec.rb index 472fbc54e807797f3faa96a2fb7260b6643978c6..3fc6efc20649070e7f5aa2b9c571a1aa727545ec 100644 --- a/spec/tooling/danger/stable_branch_spec.rb +++ b/spec/tooling/danger/stable_branch_spec.rb @@ -263,7 +263,7 @@ } }, { - 'name' => 'follow-up-e2e:package-and-test-ee', + 'name' => 'follow-up:e2e:package-and-test-ee', 'status' => 'failed', 'downstream_pipeline' => { 'id' => '456',