Extract package helpers to its own file

gitlab_component_helpers.sh relies on several
ENV variables only present in the CI.

Extracting this file will make it easier to reuse those
package functions outside of the CI.

doc/development/testing_guide/frontend_testing.md

@@ -894,6 +894,26 @@ You can generate fixtures by running:
 
 You can find generated fixtures are in `tmp/tests/frontend/fixtures-ee`.
 
+### Download fixtures
+
+We generate fixtures in GitLab CI, and store them in the package registry.
+
+The `scripts/frontend/download_fixtures.sh` script is meant to download and extract those fixtures for local use:
+
+```shell
+# Checks if a frontend fixture package exists in the gitlab-org/gitlab
+# package registry by looking at the commits on a local branch.
+# 
+# The package is downloaded and extracted if it exists
+$ scripts/frontend/download_fixtures.sh
+
+# Same as above, but only looks at the last 10 commits of the currently checked-out branch
+$ scripts/frontend/download_fixtures.sh --max-commits=10
+
+# Looks at the commits on the local master branch instead of the currently checked-out branch
+$ scripts/frontend/download_fixtures.sh --branch master
+```
+
 #### Creating new fixtures
 
 For each fixture, you can find the content of the `response` variable in the output file.

jest_resolver.js

@@ -9,7 +9,7 @@ module.exports = (request, options) => {
       console.error(
         '\x1b[1m\x1b[41m\x1b[30m %s \x1b[0m %s',
         '!',
-        `Fixture file ${request} does not exist. Did you run bin/rake frontend:fixtures?`,
+        `Fixture file ${request} does not exist. Did you run bin/rake frontend:fixtures? You can also download fixtures from the gitlab-org/gitlab package registry. See the Fixtures docs for more info.`,
       );
     }
     throw e;

scripts/frontend/download_fixtures.sh

+#!/usr/bin/env bash
+
+#
+# Downloads the most recent frontend fixtures for the current commit, going up the commit parent
+# chain up to max-commits commits (defaults to 50 commits).
+#
+
+source scripts/packages/helpers.sh
+
+print_help() {
+  echo "Usage: scripts/frontend/download_fixtures.sh [--branch <branch-name>] [--max-commits <number>]"
+  echo
+  echo "Looks for a frontend fixture package in the package registry for commits on a local branch."
+  echo
+  echo "If --branch isn't specified, the script will use the current branch as a commit reference."
+  echo "If --max-commits isn't specified, the default is 50 commits."
+
+  return
+}
+
+branch="HEAD"
+max_commits_count=50
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --branch)
+      shift
+      branch="$1"
+      ;;
+    --max-commits)
+      shift
+      max_commits_count="$1"
+      ;;
+     *)
+      print_help
+      exit
+      ;;
+  esac
+  shift
+done
+
+for commit_sha in $(git rev-list ${branch} --max-count="${max_commits_count}"); do
+  API_PACKAGES_BASE_URL=https://gitlab.com/api/v4/projects/278964/packages/generic
+  FIXTURES_PACKAGE="fixtures-${commit_sha}.tar.gz"
+  FIXTURES_PACKAGE_URL="${API_PACKAGES_BASE_URL}/fixtures/${commit_sha}/${FIXTURES_PACKAGE}"
+
+  echo "Looking for frontend fixtures for commit ${commit_sha}..."
+
+  if ! archive_doesnt_exist "${FIXTURES_PACKAGE_URL}" > /dev/null 2>&1; then
+    echo "We have found frontend fixtures at ${FIXTURES_PACKAGE_URL}!"
+
+    read_curl_package "${FIXTURES_PACKAGE_URL}" | extract_package
+
+    break
+  fi
+done

scripts/gitlab_component_helpers.sh

@@ -2,6 +2,9 @@
 
 set -euo pipefail
 
+# Generic helper functions for archives/packages
+source scripts/packages/helpers.sh
+
 export CURL_TOKEN_HEADER="${CURL_TOKEN_HEADER:-"JOB-TOKEN"}"
 
 export GITLAB_COM_CANONICAL_PROJECT_ID="278964" # https://gitlab.com/gitlab-org/gitlab
@@ -54,63 +57,6 @@ export GITLAB_ASSETS_PACKAGE_URL="${API_PACKAGES_BASE_URL}/assets/${NODE_ENV}-${
 # Fixtures constants
 export FIXTURES_PATH="tmp/tests/frontend/**/*"
 
-# Generic helper functions
-function archive_doesnt_exist() {
-  local package_url="${1}"
-
-  status=$(curl -I --silent --retry 3 --output /dev/null -w "%{http_code}" "${package_url}")
-
-  if [[ "${status}" = "200" ]]; then
-    echoinfo "The archive was found. The server returned status ${status}."
-    return 1
-  else
-    echoinfo "The archive was not found. The server returned status ${status}."
-    return 0
-  fi
-}
-
-function create_package() {
-  local archive_filename="${1}"
-  local paths_to_archive="${2}"
-  local tar_working_folder="${3:-.}"
-
-  echoinfo "Running 'tar -czvf ${archive_filename} -C ${tar_working_folder} ${paths_to_archive}'"
-  tar -czf ${archive_filename} -C ${tar_working_folder} ${paths_to_archive}
-  du -h ${archive_filename}
-}
-
-function upload_package() {
-  local archive_filename="${1}"
-  local package_url="${2}"
-  local token_header="${CURL_TOKEN_HEADER}"
-  local token="${CI_JOB_TOKEN}"
-
-  if [[ "${UPLOAD_PACKAGE_FLAG}" = "false" ]]; then
-    echoerr "The archive ${archive_filename} isn't supposed to be uploaded for this instance (${CI_SERVER_HOST}) & project (${CI_PROJECT_PATH})!"
-    exit 1
-  fi
-
-  echoinfo "Uploading ${archive_filename} to ${package_url} ..."
-  curl --fail --silent --retry 3 --header "${token_header}: ${token}" --upload-file "${archive_filename}" "${package_url}"
-}
-
-function read_curl_package() {
-  local package_url="${1}"
-
-  echoinfo "Downloading from ${package_url} ..."
-
-  curl --fail --silent --retry 3 "${package_url}"
-}
-
-function extract_package() {
-  local tar_working_folder="${1:-.}"
-  mkdir -p "${tar_working_folder}"
-
-  echoinfo "Extracting archive to ${tar_working_folder}"
-
-  tar -xz -C ${tar_working_folder} < /dev/stdin
-}
-
 # Workhorse functions
 function gitlab_workhorse_archive_doesnt_exist() {
   archive_doesnt_exist "${GITLAB_WORKHORSE_PACKAGE_URL}"

scripts/packages/helpers.sh

+#!/usr/bin/env bash
+
+source scripts/utils.sh
+
+function archive_doesnt_exist() {
+  local package_url="${1}"
+
+  status=$(curl -I --silent --retry 3 --output /dev/null -w "%{http_code}" "${package_url}")
+
+  if [[ "${status}" = "200" ]]; then
+    echoinfo "The archive was found. The server returned status ${status}."
+    return 1
+  else
+    echoinfo "The archive was not found. The server returned status ${status}."
+    return 0
+  fi
+}
+
+function create_package() {
+  local archive_filename="${1}"
+  local paths_to_archive="${2}"
+  local tar_working_folder="${3:-.}"
+
+  echoinfo "Running 'tar -czvf ${archive_filename} -C ${tar_working_folder} ${paths_to_archive}'"
+  tar -czf ${archive_filename} -C ${tar_working_folder} ${paths_to_archive}
+  du -h ${archive_filename}
+}
+
+function upload_package() {
+  local archive_filename="${1}"
+  local package_url="${2}"
+  local token_header="${CURL_TOKEN_HEADER}"
+  local token="${CI_JOB_TOKEN}"
+
+  if [[ "${UPLOAD_PACKAGE_FLAG}" = "false" ]]; then
+    echoerr "The archive ${archive_filename} isn't supposed to be uploaded for this instance (${CI_SERVER_HOST}) & project (${CI_PROJECT_PATH})!"
+    exit 1
+  fi
+
+  echoinfo "Uploading ${archive_filename} to ${package_url} ..."
+  curl --fail --silent --retry 3 --header "${token_header}: ${token}" --upload-file "${archive_filename}" "${package_url}"
+}
+
+function read_curl_package() {
+  local package_url="${1}"
+
+  echoinfo "Downloading from ${package_url} ..."
+
+  curl --fail --silent --retry 3 "${package_url}"
+}
+
+function extract_package() {
+  local tar_working_folder="${1:-.}"
+  mkdir -p "${tar_working_folder}"
+
+  echoinfo "Extracting archive to ${tar_working_folder}"
+
+  tar -xz -C ${tar_working_folder} < /dev/stdin
+}

spec/frontend/__helpers__/fixtures.js

@@ -12,7 +12,10 @@ export function getFixture(relativePath) {
     throw new ErrorWithStack(
       `Fixture file ${relativePath} does not exist.
 
-Did you run bin/rake frontend:fixtures?`,
+Did you run bin/rake frontend:fixtures? You can also download fixtures from the gitlab-org/gitlab package registry.
+
+See https://docs.gitlab.com/ee/development/testing_guide/frontend_testing.html#download-fixtures for more info.
+`,
       getFixture,
     );
   }

spec/frontend_integration/README.md

@@ -22,6 +22,8 @@ We can generate the necessary fixtures and GraphQL schema by running:
 bundle exec rake frontend:fixtures gitlab:graphql:schema:dump
 ```
 
+You can also download those fixtures from the package registry: see [download fixtures](https://docs.gitlab.com/ee/development/testing_guide/frontend_testing.html#download-fixtures) for more info.
+
 Then we can use [Jest](https://jestjs.io/) to run the frontend integration tests:
 
 ```shell