diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1340bd975e3b2a9fa46ef0fc900a5e6354d24202..6853b100ed7b9e2e8cefb16d9e12fb0fc0b659a2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,10 +41,9 @@ workflow: QA_IMAGE: "${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-ee-qa:${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}" # Also run (detached) merge request pipelines. - if: '$CI_MERGE_REQUEST_IID' - # For the 2-hourly scheduled pipelines, we set specific variables. - - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "schedule" && $FREQUENCY == "2-hourly"' + # For the maintenance scheduled pipelines, we set specific variables. + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "maintenance"' variables: - RUBY_VERSION: "2.7" CRYSTALBALL: "true" # For `$CI_DEFAULT_BRANCH` branch, create a pipeline (this includes on schedules, pushes, merges, etc.). - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' diff --git a/.gitlab/ci/pages.gitlab-ci.yml b/.gitlab/ci/pages.gitlab-ci.yml index 6f96d84b8e3733c3e799c2eaada0af8f9907eee7..1f9f57cfc22276191d6ce9410f969ca7b09c02f3 100644 --- a/.gitlab/ci/pages.gitlab-ci.yml +++ b/.gitlab/ci/pages.gitlab-ci.yml @@ -15,7 +15,7 @@ pages: - job: "compile-production-assets" - job: "compile-storybook" # `update-tests-metadata` only runs on GitLab.com's EE schedules pipelines - # while `pages` runs for all the 2-hourly schedules. + # while `pages` runs for all the maintenance scheduled pipelines. - job: "update-tests-metadata" optional: true before_script: diff --git a/.gitlab/ci/review-apps/main.gitlab-ci.yml b/.gitlab/ci/review-apps/main.gitlab-ci.yml index f3cde5d731889ca14465e6b9e1e93e1a3516acea..45ada3fd09ea626ce3fc1132225720e37e06a101 100644 --- a/.gitlab/ci/review-apps/main.gitlab-ci.yml +++ b/.gitlab/ci/review-apps/main.gitlab-ci.yml @@ -20,7 +20,7 @@ review-build-cng-env: extends: - .default-retry - .review:rules:review-build-cng - image: ${GITLAB_DEPENDENCY_PROXY}ruby:${RUBY_VERSION}-alpine3.13 + image: ${GITLAB_DEPENDENCY_PROXY}ruby:3.0-alpine3.13 stage: prepare needs: [] before_script: @@ -79,7 +79,7 @@ review-build-cng: DOMAIN: "-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}" GITLAB_HELM_CHART_REF: "a6a609a19166f00b1a7774374041cd38a9f7e20d" environment: - name: review/${CI_COMMIT_REF_SLUG}${FREQUENCY} + name: review/${CI_COMMIT_REF_SLUG}${SCHEDULE_TYPE} # No separator for SCHEDULE_TYPE so it's compatible as before and looks nice without it url: https://gitlab-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN} on_stop: review-stop auto_stop_in: 48 hours diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml index 86f5ef48522e3ede6904e558229ea57362df24a2..5c2c3d83277ef2db84b498216d276443a136b4c3 100644 --- a/.gitlab/ci/review.gitlab-ci.yml +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -8,7 +8,7 @@ review-cleanup: image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:gitlab-gcloud-helm3.5-kubectl1.17 stage: prepare environment: - name: review/${CI_COMMIT_REF_SLUG}${FREQUENCY} + name: review/${CI_COMMIT_REF_SLUG}${SCHEDULE_TYPE} # No separator for SCHEDULE_TYPE so it's compatible as before and looks nice without it action: stop before_script: - source scripts/utils.sh @@ -33,7 +33,7 @@ start-review-app-pipeline: # They need to be explicitly passed on to the child pipeline. # https://docs.gitlab.com/ee/ci/pipelines/multi_project_pipelines.html#pass-cicd-variables-to-a-downstream-pipeline-by-using-the-variables-keyword variables: - FREQUENCY: $FREQUENCY + SCHEDULE_TYPE: $SCHEDULE_TYPE DAST_RUN: $DAST_RUN trigger: include: diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index a2dd646d895131e2c5163b26871d110bee54782a..fb4016a10f0bd4bf7c2160d072bf248db4805499 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -91,11 +91,11 @@ .if-fork-merge-request: &if-fork-merge-request if: '$CI_PROJECT_NAMESPACE !~ /^gitlab(-org)?($|\/)/ && $CI_MERGE_REQUEST_IID && $CI_MERGE_REQUEST_LABELS !~ /pipeline:run-all-rspec/' -.if-default-branch-schedule-2-hourly: &if-default-branch-schedule-2-hourly - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "schedule" && $FREQUENCY == "2-hourly"' +.if-default-branch-schedule-maintenance: &if-default-branch-schedule-maintenance + if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "maintenance"' .if-default-branch-schedule-nightly: &if-default-branch-schedule-nightly - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "schedule" && $FREQUENCY == "nightly"' + if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "nightly"' .if-security-schedule: &if-security-schedule if: '$CI_PROJECT_NAMESPACE == "gitlab-org/security" && $CI_PIPELINE_SOURCE == "schedule"' @@ -106,17 +106,14 @@ .if-dot-com-ee-schedule: &if-dot-com-ee-schedule if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab" && $CI_PIPELINE_SOURCE == "schedule"' -.if-dot-com-ee-schedule-child-pipeline: &if-dot-com-ee-schedule-child-pipeline - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab" && $CI_PIPELINE_SOURCE == "parent_pipeline" && $FREQUENCY' +.if-dot-com-ee-schedule-maintenance: &if-dot-com-ee-schedule-maintenance + if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab" && $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "maintenance"' -.if-dot-com-ee-2-hourly-schedule: &if-dot-com-ee-2-hourly-schedule - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab" && $CI_PIPELINE_SOURCE == "schedule" && $FREQUENCY == "2-hourly"' +.if-dot-com-ee-schedule-nightly: &if-dot-com-ee-schedule-nightly + if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab" && $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "nightly"' -.if-dot-com-ee-nightly-schedule: &if-dot-com-ee-nightly-schedule - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab" && $CI_PIPELINE_SOURCE == "schedule" && $FREQUENCY == "nightly"' - -.if-dot-com-ee-nightly-schedule-child-pipeline: &if-dot-com-ee-nightly-schedule-child-pipeline - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab" && $CI_PIPELINE_SOURCE == "parent_pipeline" && $FREQUENCY == "nightly"' +.if-dot-com-ee-schedule-nightly-child-pipeline: &if-dot-com-ee-schedule-nightly-child-pipeline + if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab" && $CI_PIPELINE_SOURCE == "parent_pipeline" && $SCHEDULE_TYPE == "nightly"' .if-dot-com-gitlab-org-default-branch: &if-dot-com-gitlab-org-default-branch if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" && $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH' @@ -625,7 +622,7 @@ ################ .shared:rules:update-cache: rules: - - <<: *if-default-branch-schedule-2-hourly + - <<: *if-default-branch-schedule-maintenance - <<: *if-security-schedule - <<: *if-merge-request-labels-update-caches @@ -884,7 +881,7 @@ ############### .pages:rules: rules: - - <<: *if-dot-com-ee-2-hourly-schedule + - <<: *if-dot-com-ee-schedule-maintenance ############ # QA rules # @@ -1384,7 +1381,7 @@ - <<: *if-merge-request changes: *code-backstage-patterns when: always - - <<: *if-default-branch-schedule-2-hourly + - <<: *if-default-branch-schedule-maintenance - <<: *if-merge-request-labels-run-all-rspec when: always @@ -1608,13 +1605,13 @@ rules: - if: '$DAST_DISABLED || $GITLAB_FEATURES !~ /\bdast\b/' when: never - - <<: *if-dot-com-ee-nightly-schedule-child-pipeline + - <<: *if-dot-com-ee-schedule-nightly-child-pipeline .reports:rules:package_hunter-yarn: rules: - if: "$PACKAGE_HUNTER_USER == null || $PACKAGE_HUNTER_USER == ''" when: never - - <<: *if-default-branch-schedule-2-hourly + - <<: *if-default-branch-schedule-maintenance - <<: *if-merge-request changes: ["yarn.lock"] @@ -1622,7 +1619,7 @@ rules: - if: "$PACKAGE_HUNTER_USER == null || $PACKAGE_HUNTER_USER == ''" when: never - - <<: *if-default-branch-schedule-2-hourly + - <<: *if-default-branch-schedule-maintenance - <<: *if-merge-request changes: ["Gemfile.lock"] @@ -1807,12 +1804,11 @@ when: never - <<: *if-merge-request-labels-jh-contribution - .setup:rules:generate-frontend-fixtures-mapping: rules: - <<: *if-not-ee when: never - - <<: *if-dot-com-ee-2-hourly-schedule + - <<: *if-dot-com-ee-schedule-maintenance - changes: - ".gitlab/ci/setup.gitlab-ci.yml" - ".gitlab/ci/test-metadata.gitlab-ci.yml" @@ -1844,7 +1840,7 @@ rules: - <<: *if-not-ee when: never - - <<: *if-dot-com-ee-2-hourly-schedule + - <<: *if-dot-com-ee-schedule-maintenance - changes: - ".gitlab/ci/test-metadata.gitlab-ci.yml" - "scripts/rspec_helpers.sh" diff --git a/doc/development/pipelines.md b/doc/development/pipelines.md index c87c3c152075a6b53a6b84742f2947e1f4b8f653..d4395cc4e4ba6d4e42823256c8730ca57891d133 100644 --- a/doc/development/pipelines.md +++ b/doc/development/pipelines.md @@ -171,7 +171,7 @@ Our current RSpec tests parallelization setup is as follows: 1. The `retrieve-tests-metadata` job in the `prepare` stage ensures we have a `knapsack/report-master.json` file: - The `knapsack/report-master.json` file is fetched from the latest `main` pipeline which runs `update-tests-metadata` - (for now it's the 2-hourly scheduled master pipeline), if it's not here we initialize the file with `{}`. + (for now it's the 2-hourly `maintenance` scheduled master pipeline), if it's not here we initialize the file with `{}`. 1. Each `[rspec|rspec-ee] [migration|unit|integration|system|geo] n m` job are run with `knapsack rspec` and should have an evenly distributed share of tests: - It works because the jobs have access to the `knapsack/report-master.json` @@ -305,7 +305,7 @@ If these commands return `undercover: ✅ No coverage is missing in latest chang Our test suite runs against Ruby 2 in merge requests and default branch pipelines. -We do run our test suite against Ruby 3 on 2-hourly scheduled pipelines, as GitLab.com will soon run on Ruby 3. +We also run our test suite against Ruby 3 on another 2-hourly scheduled pipelines, as GitLab.com will soon run on Ruby 3. ## PostgreSQL versions testing @@ -318,12 +318,13 @@ We also run our test suite against PG11 upon specific database library changes i ### Current versions testing -| Where? | PostgreSQL version | -| ------ | ------------------ | -| MRs | 12, 11 for DB library changes | -| `main` (non-scheduled pipelines) | 12, 11 for DB library changes | -| 2-hourly scheduled pipelines | 12, 11 for DB library changes | -| `nightly` scheduled pipelines | 12, 11, 13 | +| Where? | PostgreSQL version | Ruby version | +| ------ | ------------------ | ------------ | +| Merge requests | 12 (default version), 11 for DB library changes | 2.7 (default version) | +| `master` branch commits | 12 (default version), 11 for DB library changes | 2.7 (default version) | +| `maintenance` scheduled pipelines (every 2 hours at even hour) | 12 (default version), 11 for DB library changes | 2.7 (default version) | +| `maintenance` scheduled pipelines (every 2 hours at odd hour) | 12 (default version), 11 for DB library changes | 3.0 (set in the schedule variables) | +| `nightly` scheduled pipelines | 12 (default version), 11, 13 | 2.7 (default version) | ### Long-term plan @@ -618,7 +619,7 @@ and included in `rules` definitions via [YAML anchors](../ci/yaml/yaml_optimizat | `if-default-refs` | Matches if the pipeline is for `master`, `main`, `/^[\d-]+-stable(-ee)?$/` (stable branches), `/^\d+-\d+-auto-deploy-\d+$/` (auto-deploy branches), `/^security\//` (security branches), merge requests, and tags. | Note that jobs aren't created for branches with this default configuration. | | `if-master-refs` | Matches if the current branch is `master` or `main`. | | | `if-master-push` | Matches if the current branch is `master` or `main` and pipeline source is `push`. | | -| `if-master-schedule-2-hourly` | Matches if the current branch is `master` or `main` and pipeline runs on a 2-hourly schedule. | | +| `if-master-schedule-maintenance` | Matches if the current branch is `master` or `main` and pipeline runs on a 2-hourly schedule. | | | `if-master-schedule-nightly` | Matches if the current branch is `master` or `main` and pipeline runs on a nightly schedule. | | | `if-auto-deploy-branches` | Matches if the current branch is an auto-deploy one. | | | `if-master-or-tag` | Matches if the pipeline is for the `master` or `main` branch or for a tag. | | @@ -705,7 +706,7 @@ This works well for the following reasons: - `.yarn-cache` - `.assets-compile-cache` (the key includes `${NODE_ENV}` so it's actually two different caches). 1. These cache definitions are composed of [multiple atomic caches](../ci/caching/index.md#use-multiple-caches). -1. Only the following jobs, running in 2-hourly scheduled pipelines, are pushing (that is, updating) to the caches: +1. Only the following jobs, running in 2-hourly `maintenance` scheduled pipelines, are pushing (that is, updating) to the caches: - `update-setup-test-env-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rails.gitlab-ci.yml). - `update-gitaly-binaries-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rails.gitlab-ci.yml). - `update-rubocop-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rails.gitlab-ci.yml).