diff --git a/.gitlab/ci/build-images.gitlab-ci.yml b/.gitlab/ci/build-images.gitlab-ci.yml index 3c7056a92c1c899ca39a41e7d89cf9742143f874..bb18ba12c4b0764179d90bdae2a192c57d1152bd 100644 --- a/.gitlab/ci/build-images.gitlab-ci.yml +++ b/.gitlab/ci/build-images.gitlab-ci.yml @@ -18,7 +18,7 @@ build-qa-image: - ./scripts/build_qa_image # This image is used by: -# - The `CNG` pipelines (via the `review-build-cng` job): https://gitlab.com/gitlab-org/build/CNG/-/blob/cfc67136d711e1c8c409bf8e57427a644393da2f/.gitlab-ci.yml#L335 +# - The `CNG` downstream pipelines (we pass the image tag via the `review-build-cng` job): https://gitlab.com/gitlab-org/gitlab/-/blob/c34e0834b01cd45c1f69a01b5e38dd6bc505f903/.gitlab/ci/review-apps/main.gitlab-ci.yml#L69 # - The `omnibus-gitlab` pipelines (via the `e2e:package-and-test` job): https://gitlab.com/gitlab-org/omnibus-gitlab/-/blob/dfd1ad475868fc84e91ab7b5706aa03e46dc3a86/.gitlab-ci.yml#L130 build-assets-image: extends: @@ -27,7 +27,10 @@ build-assets-image: stage: build-images needs: ["compile-production-assets"] script: - # TODO: Change the image tag to be the MD5 of assets files and skip image building if the image exists - # We'll also need to pass GITLAB_ASSETS_TAG to the trigerred omnibus-gitlab pipeline similarly to how we do it for trigerred CNG pipelines - # https://gitlab.com/gitlab-org/gitlab/issues/208389 - run_timed_command "scripts/build_assets_image" + artifacts: + expire_in: 7 days + paths: + # The `cached-assets-hash.txt` file is used in `review-build-cng-env` (`.gitlab/ci/review-apps/main.gitlab-ci.yml`) + # to pass the assets image tag to the CNG downstream pipeline. + - cached-assets-hash.txt diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index 803eeebfc4590af1749289cabe12d89bca9fad7e..d3ae3df6050b05cb04dfceb1630d0993e8b8273e 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -23,6 +23,7 @@ gitlab_assets_archive_doesnt_exist || run_timed_command "download_and_extract_gitlab_assets" fi - assets_compile_script + - echo -n "${GITLAB_ASSETS_HASH}" > "cached-assets-hash.txt" compile-production-assets: extends: @@ -38,6 +39,7 @@ compile-production-assets: # These assets are used in multiple locations: # - in `build-assets-image` job to create assets image for packaging systems # - GitLab UI for integration tests: https://gitlab.com/gitlab-org/gitlab-ui/-/blob/e88493b3c855aea30bf60baee692a64606b0eb1e/.storybook/preview-head.pug#L1 + - cached-assets-hash.txt - public/assets/ - "${WEBPACK_COMPILE_LOG_PATH}" when: always @@ -68,9 +70,6 @@ update-assets-compile-production-cache: - .assets-compile-cache-push - .shared:rules:update-cache stage: prepare - script: - - !reference [compile-production-assets, script] - - echo -n "${GITLAB_ASSETS_HASH}" > "cached-assets-hash.txt" artifacts: {} # This job's purpose is only to update the cache. update-assets-compile-test-cache: diff --git a/.gitlab/ci/package-and-test/main.gitlab-ci.yml b/.gitlab/ci/package-and-test/main.gitlab-ci.yml index 1cb123631689161cf3150e1d532466f825852632..96feca1690b8d4a2ce210f05e49a07d90441d530 100644 --- a/.gitlab/ci/package-and-test/main.gitlab-ci.yml +++ b/.gitlab/ci/package-and-test/main.gitlab-ci.yml @@ -38,23 +38,6 @@ stages: extends: - .gitlab-qa-install -.omnibus-env: - variables: - BUILD_ENV: build.env - script: - - | - SECURITY_SOURCES=$([[ ! "$CI_PROJECT_NAMESPACE" =~ ^gitlab-org\/security ]] || echo "true") - echo "SECURITY_SOURCES=${SECURITY_SOURCES:-false}" > $BUILD_ENV - echo "OMNIBUS_GITLAB_CACHE_UPDATE=${OMNIBUS_GITLAB_CACHE_UPDATE:-false}" >> $BUILD_ENV - for version_file in *_VERSION; do echo "$version_file=$(cat $version_file)" >> $BUILD_ENV; done - echo "OMNIBUS_GITLAB_RUBY3_BUILD=${OMNIBUS_GITLAB_RUBY3_BUILD:-false}" >> $BUILD_ENV - echo "OMNIBUS_GITLAB_CACHE_EDITION=${OMNIBUS_GITLAB_CACHE_EDITION:-GITLAB}" >> $BUILD_ENV - echo "Built environment file for omnibus build:" - cat $BUILD_ENV - artifacts: - reports: - dotenv: $BUILD_ENV - .update-script: script: - export QA_COMMAND="bundle exec gitlab-qa Test::Omnibus::UpdateFromPrevious $RELEASE $GITLAB_VERSION $UPDATE_TYPE -- $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS" @@ -108,9 +91,42 @@ dont-interrupt-me: trigger-omnibus-env: extends: - - .omnibus-env - .rules:omnibus-build stage: .pre + needs: + # We need this job because we need its `cached-assets-hash.txt` artifact, so that we can pass the assets image tag to the downstream omnibus-gitlab pipeline. + - pipeline: $PARENT_PIPELINE_ID + job: build-assets-image + variables: + BUILD_ENV: build.env + before_script: + - | + # This is duplicating the function from `scripts/utils.sh` since that file can be included in other projects. + function assets_image_tag() { + local cache_assets_hash_file="cached-assets-hash.txt" + + if [[ -n "${CI_COMMIT_TAG}" ]]; then + echo -n "${CI_COMMIT_REF_NAME}" + elif [[ -f "${cache_assets_hash_file}" ]]; then + echo -n "assets-hash-$(cat ${cache_assets_hash_file} | cut -c1-10)" + else + echo -n "${CI_COMMIT_SHA}" + fi + } + script: + - | + SECURITY_SOURCES=$([[ ! "$CI_PROJECT_NAMESPACE" =~ ^gitlab-org\/security ]] || echo "true") + echo "SECURITY_SOURCES=${SECURITY_SOURCES:-false}" > $BUILD_ENV + echo "OMNIBUS_GITLAB_CACHE_UPDATE=${OMNIBUS_GITLAB_CACHE_UPDATE:-false}" >> $BUILD_ENV + for version_file in *_VERSION; do echo "$version_file=$(cat $version_file)" >> $BUILD_ENV; done + echo "OMNIBUS_GITLAB_RUBY3_BUILD=${OMNIBUS_GITLAB_RUBY3_BUILD:-false}" >> $BUILD_ENV + echo "OMNIBUS_GITLAB_CACHE_EDITION=${OMNIBUS_GITLAB_CACHE_EDITION:-GITLAB}" >> $BUILD_ENV + echo "GITLAB_ASSETS_TAG=$(assets_image_tag)" >> $BUILD_ENV + echo "Built environment file for omnibus build:" + cat $BUILD_ENV + artifacts: + reports: + dotenv: $BUILD_ENV trigger-omnibus: extends: .rules:omnibus-build @@ -128,6 +144,7 @@ trigger-omnibus: GITLAB_SHELL_VERSION: $GITLAB_SHELL_VERSION GITLAB_WORKHORSE_VERSION: $GITLAB_WORKHORSE_VERSION GITLAB_VERSION: $CI_COMMIT_SHA + GITLAB_ASSETS_TAG: $GITLAB_ASSETS_TAG IMAGE_TAG: $CI_COMMIT_SHA TOP_UPSTREAM_SOURCE_PROJECT: $CI_PROJECT_PATH SECURITY_SOURCES: $SECURITY_SOURCES diff --git a/.gitlab/ci/qa.gitlab-ci.yml b/.gitlab/ci/qa.gitlab-ci.yml index bd587cb441842b5461552ea0d5fc74cfcc6a1d09..b365827b1c3d20cd0b984ccc5e13a9cfe0a5b988 100644 --- a/.gitlab/ci/qa.gitlab-ci.yml +++ b/.gitlab/ci/qa.gitlab-ci.yml @@ -75,6 +75,8 @@ e2e:package-and-test: - build-qa-image - e2e-test-pipeline-generate variables: + # This is needed by `trigger-omnibus-env` (`.gitlab/ci/package-and-test/main.gitlab-ci.yml`). + PARENT_PIPELINE_ID: $CI_PIPELINE_ID SKIP_MESSAGE: Skipping package-and-test due to mr containing only quarantine changes! RELEASE: "${REGISTRY_HOST}/${REGISTRY_GROUP}/build/omnibus-gitlab-mirror/gitlab-ee:${CI_COMMIT_SHA}" GITLAB_QA_IMAGE: "${CI_REGISTRY_IMAGE}/gitlab-ee-qa:${CI_COMMIT_SHA}" diff --git a/.gitlab/ci/review-apps/main.gitlab-ci.yml b/.gitlab/ci/review-apps/main.gitlab-ci.yml index 1f801e0d803efa0714f5dedcee6608f7dc00e676..a71adb4fe832125aeeac9026b2ab8e4751b7e8c7 100644 --- a/.gitlab/ci/review-apps/main.gitlab-ci.yml +++ b/.gitlab/ci/review-apps/main.gitlab-ci.yml @@ -34,18 +34,24 @@ review-build-cng-env: - .review:rules:review-build-cng image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}ruby:3.0-alpine3.13 stage: prepare - needs: [] + needs: + # We need this job because we need its `cached-assets-hash.txt` artifact, so that we can pass the assets image tag to the downstream CNG pipeline. + - pipeline: $PARENT_PIPELINE_ID + job: build-assets-image + variables: + BUILD_ENV: build.env before_script: - source ./scripts/utils.sh - install_gitlab_gem script: - - 'ruby -r./scripts/trigger-build.rb -e "puts Trigger.variables_for_env_file(Trigger::CNG.new.variables)" > build.env' - - cat build.env + - 'ruby -r./scripts/trigger-build.rb -e "puts Trigger.variables_for_env_file(Trigger::CNG.new.variables)" > $BUILD_ENV' + - echo "GITLAB_ASSETS_TAG=$(assets_image_tag)" >> $BUILD_ENV + - cat $BUILD_ENV artifacts: reports: - dotenv: build.env + dotenv: $BUILD_ENV paths: - - build.env + - $BUILD_ENV expire_in: 7 days when: always diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml index aefa96da159eb73b98b16cadf9e80127e529c458..199f564464dd7273ea95cb1bd0ad02e4ca479cee 100644 --- a/.gitlab/ci/review.gitlab-ci.yml +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -33,6 +33,8 @@ start-review-app-pipeline: # They need to be explicitly passed on to the child pipeline. # https://docs.gitlab.com/ee/ci/pipelines/multi_project_pipelines.html#pass-cicd-variables-to-a-downstream-pipeline-by-using-the-variables-keyword variables: + # This is needed by `review-build-cng-env` (`.gitlab/ci/review-apps/main.gitlab-ci.yml`). + PARENT_PIPELINE_ID: $CI_PIPELINE_ID SCHEDULE_TYPE: $SCHEDULE_TYPE DAST_RUN: $DAST_RUN SKIP_MESSAGE: Skipping review-app due to mr containing only quarantine changes! diff --git a/Dockerfile.assets b/Dockerfile.assets index 403d16cc4ab1beae562bfefa14e9cacd0ac76942..ba69a614e888fc12995c0fe53ba7abd46ad40cb1 100644 --- a/Dockerfile.assets +++ b/Dockerfile.assets @@ -1,4 +1,4 @@ # Simple container to store assets for later use FROM scratch -ADD public/assets /assets/ +COPY public/assets /assets/ CMD /bin/true diff --git a/scripts/build_assets_image b/scripts/build_assets_image index 8aa6526061adaca9eeb3ce7e3369b389839c4490..7482a170fe7074c6571baa5995e9847f6d74e1dd 100755 --- a/scripts/build_assets_image +++ b/scripts/build_assets_image @@ -1,36 +1,82 @@ +#!/bin/sh + +. scripts/utils.sh + # Exit early if we don't want to build the image -if [[ "${BUILD_ASSETS_IMAGE}" != "true" ]] +if [ "${BUILD_ASSETS_IMAGE}" != "true" ] then exit 0 fi +get_repository_id() { + repository_name="${1}" + repositories_url="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/registry/repositories" + + curl --header "PRIVATE-TOKEN: ${PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE}" "${repositories_url}" | jq "map(select(.name == \"${repository_name}\")) | .[0].id" +} + # Generate the image name based on the project this is being run in ASSETS_IMAGE_NAME="gitlab-assets-ce" + # `dev.gitlab-org` still has gitlab-ee. -if [[ "${CI_PROJECT_NAME}" == "gitlab" ]] || [[ "${CI_PROJECT_NAME}" == "gitlab-ee" ]] +if [ "${CI_PROJECT_NAME}" = "gitlab" ] || [ "${CI_PROJECT_NAME}" = "gitlab-ee" ] then ASSETS_IMAGE_NAME="gitlab-assets-ee" fi -ASSETS_IMAGE_PATH=${CI_REGISTRY}/${CI_PROJECT_PATH}/${ASSETS_IMAGE_NAME} +ASSETS_IMAGE_PATH="${CI_REGISTRY}/${CI_PROJECT_PATH}/${ASSETS_IMAGE_NAME}" +COMMIT_ASSETS_HASH_TAG="$(assets_image_tag)" +COMMIT_ASSETS_HASH_DESTINATION="${ASSETS_IMAGE_PATH}:${COMMIT_ASSETS_HASH_TAG}" -mkdir -p assets_container.build/public -cp -r public/assets assets_container.build/public/ -cp Dockerfile.assets assets_container.build/ +DESTINATIONS="--destination=${COMMIT_ASSETS_HASH_DESTINATION}" + +SKIP_ASSETS_IMAGE_BUILDING_IF_ALREADY_EXIST="true" -COMMIT_REF_SLUG_DESTINATION=${ASSETS_IMAGE_PATH}:${CI_COMMIT_REF_SLUG} +# Also tag the image with GitLab version, if running on a tag pipeline +# (and thus skip the performance optimization in that case), for back-compatibility. +if [ -n "${CI_COMMIT_TAG}" ]; then + COMMIT_REF_NAME_DESTINATION="${ASSETS_IMAGE_PATH}:${CI_COMMIT_REF_NAME}" + DESTINATIONS="$DESTINATIONS --destination=$COMMIT_REF_NAME_DESTINATION" + SKIP_ASSETS_IMAGE_BUILDING_IF_ALREADY_EXIST="false" +fi -COMMIT_SHA_DESTINATION=${ASSETS_IMAGE_PATH}:${CI_COMMIT_SHA} -COMMIT_REF_NAME_DESTINATION=${ASSETS_IMAGE_PATH}:${CI_COMMIT_REF_NAME} +# The auto-deploy branch process still fetch assets image tagged with $CI_COMMIT_SHA, +# so we need to push the image with it (and thus skip the performance optimization in that case), +# for back-compatibility. +if echo "${CI_COMMIT_BRANCH}" | grep -Eq "^[0-9]+-[0-9]+-auto-deploy-[0-9]+$"; then + COMMIT_SHA_DESTINATION=${ASSETS_IMAGE_PATH}:${CI_COMMIT_SHA} + DESTINATIONS="$DESTINATIONS --destination=$COMMIT_SHA_DESTINATION" + SKIP_ASSETS_IMAGE_BUILDING_IF_ALREADY_EXIST="false" +fi -DESTINATIONS="--destination=$COMMIT_REF_SLUG_DESTINATION --destination=$COMMIT_SHA_DESTINATION" +if [ "${SKIP_ASSETS_IMAGE_BUILDING_IF_ALREADY_EXIST}" = "true" ] && [ -n "${PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE}" ]; then + echoinfo "Checking if the ${COMMIT_ASSETS_HASH_DESTINATION} image exists..." + repository_id=$(get_repository_id "${ASSETS_IMAGE_NAME}") -# Also tag the image with GitLab version, if running on a tag pipeline, so -# other projects can simply use that instead of computing the slug. -if [ -n "$CI_COMMIT_TAG" ]; then - DESTINATIONS="$DESTINATIONS --destination=$COMMIT_REF_NAME_DESTINATION" + if [ -n "${repository_id}" ]; then + api_image_url="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/registry/repositories/${repository_id}/tags/${COMMIT_ASSETS_HASH_TAG}" + echoinfo "api_image_url: ${api_image_url}" + + if test_url "${api_image_url}" "--header \"PRIVATE-TOKEN: ${PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE}\""; then + echosuccess "Image ${COMMIT_ASSETS_HASH_DESTINATION} already exists, no need to rebuild it." + exit 0 + else + echoinfo "Image ${COMMIT_ASSETS_HASH_DESTINATION} doesn't exist, we'll need to build it." + fi + else + echoerr "Repository ID couldn't be found for the '${ASSETS_IMAGE_NAME}' image!" + fi +else + echoinfo "The 'PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE' variable is not present, so we cannot check if the image already exists." fi -echo "building assets image for destinations: $DESTINATIONS" +mkdir -p assets_container.build/public +cp -r public/assets assets_container.build/public/ +cp Dockerfile.assets assets_container.build/ + +echo "Building assets image for destinations: ${DESTINATIONS}" -/kaniko/executor --context=assets_container.build --dockerfile=assets_container.build/Dockerfile.assets $DESTINATIONS +/kaniko/executor \ + --context="assets_container.build" \ + --dockerfile="assets_container.build/Dockerfile.assets" \ + ${DESTINATIONS} diff --git a/scripts/trigger-build.rb b/scripts/trigger-build.rb index 897ca9f473e32ec456e9c119a2362da85270ba7f..8dfab8dd2ebbaac0f9cdb20f941a76e50c284b94 100755 --- a/scripts/trigger-build.rb +++ b/scripts/trigger-build.rb @@ -160,6 +160,8 @@ def version_file_variables end class CNG < Base + ASSETS_HASH = "cached-assets-hash.txt" + def variables # Delete variables that aren't useful when using native triggers. super.tap do |hash| @@ -187,7 +189,6 @@ def extra_variables "TRIGGER_BRANCH" => ref, "GITLAB_VERSION" => ENV['CI_COMMIT_SHA'], "GITLAB_TAG" => ENV['CI_COMMIT_TAG'], # Always set a value, even an empty string, so that the downstream pipeline can correctly check it. - "GITLAB_ASSETS_TAG" => ENV['CI_COMMIT_TAG'] ? ENV['CI_COMMIT_REF_NAME'] : ENV['CI_COMMIT_SHA'], "FORCE_RAILS_IMAGE_BUILDS" => 'true', "CE_PIPELINE" => Trigger.ee? ? nil : "true", # Always set a value, even an empty string, so that the downstream pipeline can correctly check it. "EE_PIPELINE" => Trigger.ee? ? "true" : nil # Always set a value, even an empty string, so that the downstream pipeline can correctly check it. diff --git a/scripts/utils.sh b/scripts/utils.sh index ea2b390f24948e7b94f79693f17483c059b1fc51..dae65ac8156450bcbf47d0ddc08373806043c0c3 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -15,9 +15,11 @@ function retry() { function test_url() { local url="${1}" + local curl_args="${2}" local status + local cmd="curl ${curl_args} --output /dev/null -L -s -w ''%{http_code}'' \"${url}\"" - status=$(curl --output /dev/null -L -s -w ''%{http_code}'' "${url}") + status=$(eval "${cmd}") if [[ $status == "200" ]]; then return 0 @@ -203,3 +205,16 @@ function danger_as_local() { # We need to base SHA to help danger determine the base commit for this shallow clone. bundle exec danger dry_run --fail-on-errors=true --verbose --base="${CI_MERGE_REQUEST_DIFF_BASE_SHA}" --head="${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA:-$CI_COMMIT_SHA}" --dangerfile="${DANGER_DANGERFILE:-Dangerfile}" } + +# We're inlining this function in `.gitlab/ci/package-and-test/main.gitlab-ci.yml` since this file can be included in other projects. +function assets_image_tag() { + local cache_assets_hash_file="cached-assets-hash.txt" + + if [[ -n "${CI_COMMIT_TAG}" ]]; then + echo -n "${CI_COMMIT_REF_NAME}" + elif [[ -f "${cache_assets_hash_file}" ]]; then + echo -n "assets-hash-$(cat ${cache_assets_hash_file} | cut -c1-10)" + else + echo -n "${CI_COMMIT_SHA}" + fi +} diff --git a/spec/scripts/trigger-build_spec.rb b/spec/scripts/trigger-build_spec.rb index 9032ba85b9fcac6585d58d0959cdb68da912ce9a..ebf05167428b69e9926587e54495f3066352961f 100644 --- a/spec/scripts/trigger-build_spec.rb +++ b/spec/scripts/trigger-build_spec.rb @@ -319,28 +319,6 @@ def ref_param_name end end - describe "GITLAB_ASSETS_TAG" do - context 'when CI_COMMIT_TAG is set' do - before do - stub_env('CI_COMMIT_TAG', 'v1.0') - end - - it 'sets GITLAB_ASSETS_TAG to CI_COMMIT_REF_NAME' do - expect(subject.variables['GITLAB_ASSETS_TAG']).to eq(env['CI_COMMIT_REF_NAME']) - end - end - - context 'when CI_COMMIT_TAG is nil' do - before do - stub_env('CI_COMMIT_TAG', nil) - end - - it 'sets GITLAB_ASSETS_TAG to CI_COMMIT_SHA' do - expect(subject.variables['GITLAB_ASSETS_TAG']).to eq(env['CI_COMMIT_SHA']) - end - end - end - describe "CE_PIPELINE" do context 'when Trigger.ee? is true' do before do