Merge branch 'use-repo-from-artifacts-in-frontend-jobs' into 'master'

Use the repo from artifacts in frontend jobs

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/158197



Merged-by: Lin Jen-Shin <jen-shin@gitlab.com>
Approved-by: Lin Jen-Shin <jen-shin@gitlab.com>
Reviewed-by: Rémy Coutable <remy@rymai.me>
Reviewed-by: Lin Jen-Shin <jen-shin@gitlab.com>
Co-authored-by: Rémy Coutable <remy@rymai.me>

(cherry picked from commit 2f83fd5b)

6b02c833 Use the repo from artifacts in frontend jobs
34c0263b Move frontend test to new test-frontend stage
33e3b22c ci: Don't use repo from artifacts for critical jobs

Co-authored-by: Lin Jen-Shin <jen-shin@gitlab.com>

.gitlab-ci.yml

@@ -5,6 +5,7 @@ stages:
   - build-images
   - fixtures
   - lint
+  - test-frontend
   - test
   - post-test
   - review

.gitlab/ci/frontend.gitlab-ci.yml

+.with-fixtures-needs:
+  needs:
+    - "rspec-all frontend_fixture"
+
+.with-graphql-schema-dump-needs:
+  needs:
+    - "graphql-schema-dump"
+
 .compile-assets-base:
   extends:
     - .default-retry
@@ -83,7 +91,6 @@ update-assets-compile-production-cache:
     - .update-cache-base
     - .assets-compile-cache-push
     - .shared:rules:update-cache
-  stage: prepare
   artifacts: {}  # This job's purpose is only to update the cache.
 
 update-assets-compile-test-cache:
@@ -92,7 +99,6 @@ update-assets-compile-test-cache:
     - .update-cache-base
     - .assets-compile-cache-push
     - .shared:rules:update-cache
-  stage: prepare
   artifacts: {}  # This job's purpose is only to update the cache.
 
 update-storybook-yarn-cache:
@@ -136,8 +142,14 @@ retrieve-frontend-fixtures:
     - .default-before_script
     - .ruby-cache
     - .use-pg14
+    - .repo-from-artifacts
   stage: fixtures
-  needs: ["setup-test-env", "retrieve-tests-metadata", "retrieve-frontend-fixtures"]
+  needs:
+    - "setup-test-env"
+    - "retrieve-tests-metadata"
+    - "retrieve-frontend-fixtures"
+    # it's ok to wait for the repo artifact as we're waiting for setup-test-env (which takes longer than clone-gitlab-repo) anyway
+    - !reference [.repo-from-artifacts, needs]
   variables:
     # Don't add `CRYSTALBALL: "false"` here as we're enabling Crystalball for scheduled pipelines (in `.gitlab-ci.yml`), so that we get coverage data
     # for the `frontend fixture RSpec files` that will be added to the Crystalball mapping in `update-tests-metadata`.
@@ -180,18 +192,21 @@ upload-frontend-fixtures:
   variables:
     SETUP_DB: "false"
   extends:
-    - .frontend-fixtures-base
+    - .default-retry
+    - .default-before_script
+    - .repo-from-artifacts
     - .frontend:rules:upload-frontend-fixtures
   stage: fixtures
-  needs: ["rspec-all frontend_fixture"]
+  needs:
+    # it's ok to wait for the repo artifact as we're waiting for the fixtures (which wait for the repo artifact) anyway
+    - !reference [.repo-from-artifacts, needs]
+    - !reference [.with-fixtures-needs, needs]
   script:
-    - source scripts/utils.sh
     - source scripts/gitlab_component_helpers.sh
     - export_fixtures_sha_for_upload
     - 'fixtures_archive_doesnt_exist || { echoinfo "INFO: Exiting early as package exists."; exit 0; }'
     - run_timed_command "create_fixtures_package"
     - run_timed_command "upload_fixtures_package"
-  artifacts: {}
 
 graphql-schema-dump:
   variables:
@@ -225,18 +240,7 @@ graphql-schema-dump:
   before_script:
     - !reference [.default-before_script, before_script]
     - yarn_install_script
-  stage: test
-
-.vue3:
-  variables:
-    VUE_VERSION: 3
-    NODE_OPTIONS: --max-old-space-size=7680
-  allow_failure: true
-
-.jest-base:
-  extends: .frontend-test-base
-  script:
-    - run_timed_command "yarn jest:ci:without-fixtures"
+  stage: test-frontend
 
 jest-build-cache:
   extends:
@@ -260,18 +264,28 @@ jest-build-cache:
     # they exit with 1, so as not to break master and other pipelines.
     exit_codes: 1
 
+.vue3:
+  variables:
+    VUE_VERSION: 3
+    NODE_OPTIONS: --max-old-space-size=7680
+  allow_failure: true
+
+.with-jest-build-cache-vue3-needs:
+  needs:
+    - job: jest-build-cache-vue3
+      optional: true
+
 jest-build-cache-vue3:
   extends:
     - jest-build-cache
     - .frontend:rules:jest-vue3
     - .vue3
 
-jest-with-fixtures:
+jest:
   extends:
-    - .jest-base
+    - .frontend-test-base
     - .frontend:rules:jest
   needs:
-    - "rspec-all frontend_fixture"
     - job: jest-build-cache
       optional: true
   artifacts:
@@ -284,47 +298,42 @@ jest-with-fixtures:
       - tmp/tests/frontend/
     reports:
       junit: junit_jest.xml
-  parallel: 2
+  parallel: 11
   script:
-    - run_timed_command "yarn jest:ci:with-fixtures"
+    - run_timed_command "yarn jest:ci:without-fixtures"
 
-jest:
+jest-with-fixtures:
   extends:
-    - .jest-base
+    - jest
+    - .repo-from-artifacts
     - .frontend:rules:jest
   needs:
-    - job: jest-build-cache
-      optional: true
-  artifacts:
-    name: coverage-frontend
-    expire_in: 31d
-    when: always
-    paths:
-      - coverage-frontend/
-      - junit_jest.xml
-      - tmp/tests/frontend/
-    reports:
-      junit: junit_jest.xml
-  parallel: 11
+    - !reference [jest, needs]
+    # it's ok to wait for the repo artifact as we're waiting for the fixtures (which wait for the repo artifact) anyway
+    - !reference [.repo-from-artifacts, needs]
+    - !reference [.with-fixtures-needs, needs]
+  parallel: 2
+  script:
+    - run_timed_command "yarn jest:ci:with-fixtures"
 
-jest-with-fixtures vue3:
+jest vue3:
   extends:
-    - jest-with-fixtures
+    - jest
     - .frontend:rules:jest-vue3
     - .vue3
   needs:
-    - "rspec-all frontend_fixture"
-    - job: jest-build-cache-vue3
-      optional: true
+    - !reference [.with-jest-build-cache-vue3-needs, needs]
 
-jest vue3:
+jest-with-fixtures vue3:
   extends:
-    - jest
+    - jest-with-fixtures
     - .frontend:rules:jest-vue3
     - .vue3
   needs:
-    - job: jest-build-cache-vue3
-      optional: true
+    - !reference ["jest vue3", needs]
+    # it's ok to wait for the repo artifact as we're waiting for the fixtures (which wait for the repo artifact) anyway
+    - !reference [.repo-from-artifacts, needs]
+    - !reference [.with-fixtures-needs, needs]
 
 jest predictive:
   extends:
@@ -349,16 +358,25 @@ jest-with-fixtures predictive:
 jest-integration:
   extends:
     - .frontend-test-base
+    - .repo-from-artifacts
     - .frontend:rules:jest-integration
   script:
     - run_timed_command "yarn jest:integration --ci"
-  needs: ["rspec-all frontend_fixture", "graphql-schema-dump"]
+  needs:
+    # it's ok to wait for the repo artifact as we're waiting for the fixtures (which wait for the repo artifact) anyway
+    - !reference [.repo-from-artifacts, needs]
+    - !reference [.with-fixtures-needs, needs]
+    - !reference [.with-graphql-schema-dump-needs, needs]
 
 jest-snapshot-vue3:
   extends:
-    - .jest-base
+    - .frontend-test-base
+    - .repo-from-artifacts
     - .frontend:rules:jest-snapshot-vue3
-  needs: ["rspec-all frontend_fixture"]
+  needs:
+    # it's ok to wait for the repo artifact as we're waiting for the fixtures (which wait for the repo artifact) anyway
+    - !reference [.repo-from-artifacts, needs]
+    - !reference [.with-fixtures-needs, needs]
   variables:
     VUE_VERSION: 3
     JEST_REPORT: jest-test-report.json
@@ -381,7 +399,6 @@ jest-snapshot-vue3:
         echo 'All snapshot tests passed! Exiting 0...'
         exit 0
       fi
-
   artifacts:
     name: snapshot_tests
     expire_in: 31d
@@ -395,8 +412,10 @@ coverage-frontend:
     - .default-retry
     - .default-utils-before_script
     - .yarn-cache
+    - .repo-from-artifacts
     - .frontend:rules:coverage-frontend
   needs:
+    - !reference [.repo-from-artifacts, needs]
     - job: "jest"
       optional: true
     - job: "jest-with-fixtures"
@@ -427,9 +446,9 @@ webpack-dev-server:
     - .default-retry
     - .default-utils-before_script
     - .yarn-cache
+    - .repo-from-artifacts
     - .frontend:rules:default-frontend-jobs
-  stage: test
-  needs: []
+  stage: test-frontend
   variables:
     WEBPACK_MEMORY_TEST: "true"
     WEBPACK_VENDOR_DLL: "true"
@@ -443,44 +462,24 @@ webpack-dev-server:
     paths:
       - webpack-dev-server.json
 
-bundle-size-review:
-  extends:
-    - .default-retry
-    - .default-utils-before_script
-    - .assets-compile-cache
-    - .frontend:rules:bundle-size-review
-  image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:danger
-  stage: test
-  needs: []
-  script:
-    - yarn_install_script
-    - scripts/bundle_size_review
-  artifacts:
-    when: always
-    name: bundle-size-review
-    expire_in: 31d
-    paths:
-      - bundle-size-review/
-
-.compile-storybook-base:
+compile-storybook:
   extends:
     - .frontend-test-base
     - .storybook-yarn-cache
-  script:
-    - yarn_install_script_storybook
-    - run_timed_command "yarn run storybook:build"
-  needs: ["graphql-schema-dump"]
-
-compile-storybook:
-  extends:
-    - .compile-storybook-base
+    - .repo-from-artifacts
     - .frontend:rules:compile-storybook
+  stage: pages
   needs:
-    - !reference [.compile-storybook-base, needs]
-    - job: "rspec-all frontend_fixture"
+    # it's ok to wait for the repo artifact as we're waiting for the fixtures (which wait for the repo artifact) anyway
+    - !reference [.repo-from-artifacts, needs]
+    - !reference [.with-fixtures-needs, needs]
+    - !reference [.with-graphql-schema-dump-needs, needs]
   artifacts:
     name: storybook
     expire_in: 31d
     when: always
     paths:
       - storybook/public
+  script:
+    - yarn_install_script_storybook
+    - run_timed_command "yarn run storybook:build"

.gitlab/ci/global.gitlab-ci.yml

@@ -28,7 +28,8 @@
   needs:
     # If the job extending this also defines `needs`, make sure to update
     # its `needs` to include `clone-gitlab-repo` because it'll be overridden.
-    - clone-gitlab-repo
+    - job: clone-gitlab-repo
+      optional: true # Optional so easier to switch in between
 
 .production:
   variables:

.gitlab/ci/includes/gitlab-com/danger-review.gitlab-ci.yml

@@ -2,6 +2,7 @@ include:
   - component: ${CI_SERVER_FQDN}/gitlab-org/components/danger-review/danger-review@1.4.1
     inputs:
       job_image: "${DEFAULT_CI_IMAGE}"
+      job_stage: "preflight"
       # By default DANGER_DANGERFILE_PREFIX is not defined but allows JiHu to
       # use a different prefix.
       # See https://jihulab.com/gitlab-cn/gitlab/-/blob/main-jh/jh/.gitlab-ci.yml

.gitlab/ci/preflight.gitlab-ci.yml

@@ -115,3 +115,21 @@ pipeline-tier-3:
   extends:
     - .pipeline-tier-base
     - .preflight:rules:pipeline-tier-3
+
+bundle-size-review:
+  extends:
+    - .default-retry
+    - .default-utils-before_script
+    - .assets-compile-cache
+    - .repo-from-artifacts
+    - .frontend:rules:bundle-size-review
+  stage: preflight
+  script:
+    - yarn_install_script
+    - scripts/bundle_size_review
+  artifacts:
+    when: always
+    name: bundle-size-review
+    expire_in: 31d
+    paths:
+      - bundle-size-review/

.gitlab/ci/qa.gitlab-ci.yml

@@ -103,6 +103,7 @@ qa:metadata-lint:
   extends:
     - .qa-job-base
     - .qa:rules:metadata-lint
+  stage: lint
   variables:
     QA_EXPORT_TEST_METRICS: "false"
     # Disable warnings in browserslist which can break on backports

.gitlab/ci/rails.gitlab-ci.yml

@@ -1367,7 +1367,6 @@ fail-pipeline-early:
   stage: test
   needs:
     - !reference [.rspec-base-needs, needs]
-    - job: "compile-test-assets"
     - job: "detect-previous-failed-tests"
   script:
     - !reference [.base-script, script]

.gitlab/ci/rails/shared.gitlab-ci.yml

@@ -68,10 +68,10 @@ include:
 
 .rspec-base-needs:
   needs:
-    - job: "clone-gitlab-repo"
-      optional: true  # Optional so easier to switch in between
+    - !reference [.repo-from-artifacts, needs]
     - job: "setup-test-env"
     - job: "retrieve-tests-metadata"
+    - job: "compile-test-assets"
 
 .rspec-base:
   extends:
@@ -89,7 +89,6 @@ include:
     EVENT_PROF: "sql.active_record"
   needs:
     - !reference [.rspec-base-needs, needs]
-    - job: "compile-test-assets"
     - job: "detect-tests"
       optional: true
   script:

.gitlab/ci/setup.gitlab-ci.yml

@@ -91,7 +91,7 @@ verify-tests-yml:
   extends:
     - .setup:rules:verify-tests-yml
   image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}ruby:${RUBY_VERSION}-alpine3.20
-  stage: test
+  stage: preflight
   needs: []
   script:
     - source scripts/utils.sh