Merge branch '351035-research-streamlining-or-eliminating-setup-test-env' into 'master'

ci: Build workhorse binaries and upload/download as generic package

See merge request gitlab-org/gitlab!79766

.gitlab-ci.yml

@@ -69,6 +69,9 @@ variables:
   GET_SOURCES_ATTEMPTS: "3"
   DEBIAN_VERSION: "bullseye"
 
+  TMP_TEST_FOLDER: "${CI_PROJECT_DIR}/tmp/tests"
+  GITLAB_WORKHORSE_FOLDER: "gitlab-workhorse"
+  TMP_TEST_GITLAB_WORKHORSE_PATH: "${TMP_TEST_FOLDER}/${GITLAB_WORKHORSE_FOLDER}"
   KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/report-master.json
   FLAKY_RSPEC_SUITE_REPORT_PATH: rspec/flaky/report-suite.json
   RSPEC_TESTS_MAPPING_PATH: crystalball/mapping.json

.gitlab/ci/rails.gitlab-ci.yml

@@ -183,7 +183,7 @@
 # rspec job parallel configs
 ############################
 
-#######################################################
+###############################################################
 # EE/FOSS: default refs (MRs, default branch, schedules) jobs #
 setup-test-env:
   extends:
@@ -194,40 +194,53 @@ setup-test-env:
   variables:
     SETUP_DB: "false"
   script:
+    - source scripts/gitlab_workhorse_component_helpers.sh
+    - run_timed_command "download_and_extract_gitlab_workhorse_package" || true
     - run_timed_command "scripts/setup-test-env"
+    - run_timed_command "select_gitlab_workhorse_essentials"
     - echo -e "\e[0Ksection_start:`date +%s`:gitaly-test-build[collapsed=true]\r\e[0KCompiling Gitaly binaries"
     - run_timed_command "scripts/gitaly-test-build"  # Do not use 'bundle exec' here
     - echo -e "\e[0Ksection_end:`date +%s`:gitaly-test-build\r\e[0K"
-
   artifacts:
     expire_in: 7d
     paths:
       - config/secrets.yml
-      - tmp/tests/gitaly/_build/bin/
-      - tmp/tests/gitaly/_build/deps/git/install
-      - tmp/tests/gitaly/config.toml
-      - tmp/tests/gitaly/gitaly2.config.toml
-      - tmp/tests/gitaly/internal/
-      - tmp/tests/gitaly/internal_gitaly2/
-      - tmp/tests/gitaly/internal_sockets/
-      - tmp/tests/gitaly/Makefile
-      - tmp/tests/gitaly/praefect.config.toml
-      - tmp/tests/gitaly/ruby/
-      - tmp/tests/gitlab-elasticsearch-indexer/bin/gitlab-elasticsearch-indexer
-      - tmp/tests/gitlab-shell/
-      - tmp/tests/gitlab-test-fork/
-      - tmp/tests/gitlab-test-fork_bare/
-      - tmp/tests/gitlab-test/
-      - tmp/tests/gitlab-workhorse/gitlab-zip-metadata
-      - tmp/tests/gitlab-workhorse/gitlab-zip-cat
-      - tmp/tests/gitlab-workhorse/gitlab-workhorse
-      - tmp/tests/gitlab-workhorse/gitlab-resize-image
-      - tmp/tests/gitlab-workhorse/config.toml
-      - tmp/tests/gitlab-workhorse/WORKHORSE_TREE
-      - tmp/tests/repositories/
-      - tmp/tests/second_storage/
+      - ${TMP_TEST_FOLDER}/gitaly/_build/bin/
+      - ${TMP_TEST_FOLDER}/gitaly/_build/deps/git/install
+      - ${TMP_TEST_FOLDER}/gitaly/config.toml
+      - ${TMP_TEST_FOLDER}/gitaly/gitaly2.config.toml
+      - ${TMP_TEST_FOLDER}/gitaly/internal/
+      - ${TMP_TEST_FOLDER}/gitaly/internal_gitaly2/
+      - ${TMP_TEST_FOLDER}/gitaly/internal_sockets/
+      - ${TMP_TEST_FOLDER}/gitaly/Makefile
+      - ${TMP_TEST_FOLDER}/gitaly/praefect.config.toml
+      - ${TMP_TEST_FOLDER}/gitaly/ruby/
+      - ${TMP_TEST_FOLDER}/gitlab-elasticsearch-indexer/bin/gitlab-elasticsearch-indexer
+      - ${TMP_TEST_FOLDER}/gitlab-shell/
+      - ${TMP_TEST_FOLDER}/gitlab-test-fork/
+      - ${TMP_TEST_FOLDER}/gitlab-test-fork_bare/
+      - ${TMP_TEST_FOLDER}/gitlab-test/
+      - ${TMP_TEST_FOLDER}/repositories/
+      - ${TMP_TEST_FOLDER}/second_storage/
+      - ${TMP_TEST_GITLAB_WORKHORSE_PATH}/
     when: always
 
+build-components:
+  extends:
+    - setup-test-env
+    - .rails:rules:build-components
+  script:
+    - source scripts/gitlab_workhorse_component_helpers.sh
+    - 'gitlab_workhorse_package_doesnt_exist || { echoinfo "INFO: Exiting early as package exists."; exit 0; }'
+    - run_timed_command "scripts/setup-test-env"
+    - run_timed_command "select_gitlab_workhorse_essentials"
+    - run_timed_command "create_gitlab_workhorse_package"
+    - run_timed_command "upload_gitlab_workhorse_package"
+  artifacts:
+    expire_in: 7d
+    paths:
+      - ${TMP_TEST_GITLAB_WORKHORSE_PATH}/
+
 update-setup-test-env-cache:
   extends:
     - setup-test-env

.gitlab/ci/rules.gitlab-ci.yml

@@ -876,6 +876,16 @@
 ###############
 # Rails rules #
 ###############
+.rails:rules:build-components:
+  rules:
+    - <<: *if-dot-com-ee-schedule
+    - <<: *if-dot-com-gitlab-org-default-branch
+      changes:
+        - "workhorse/**/*"
+    - <<: *if-dot-com-gitlab-org-merge-request
+      when: manual
+      allow_failure: true
+
 .rails:rules:setup-test-env:
   rules:
     - changes: *setup-test-env-patterns

scripts/gitlab_workhorse_component_helpers.sh

+#!/usr/bin/env bash
+
+set -euo pipefail
+
+export CURL_TOKEN_HEADER="${CURL_TOKEN_HEADER:-"JOB-TOKEN"}"
+export GITLAB_WORKHORSE_BINARIES_LIST="gitlab-resize-image gitlab-zip-cat gitlab-zip-metadata gitlab-workhorse"
+export GITLAB_WORKHORSE_PACKAGE_FILES_LIST="${GITLAB_WORKHORSE_BINARIES_LIST} WORKHORSE_TREE"
+export GITLAB_WORKHORSE_TREE=${GITLAB_WORKHORSE_TREE:-$(git rev-parse HEAD:workhorse)}
+export GITLAB_WORKHORSE_PACKAGE="workhorse-${GITLAB_WORKHORSE_TREE}.tar.gz"
+export GITLAB_WORKHORSE_PACKAGE_URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${GITLAB_WORKHORSE_FOLDER}/${GITLAB_WORKHORSE_TREE}/${GITLAB_WORKHORSE_PACKAGE}"
+
+function gitlab_workhorse_archive_doesnt_exist() {
+  local package_url="${GITLAB_WORKHORSE_PACKAGE_URL}"
+
+  status=$(curl -I --silent --retry 3 --output /dev/null -w "%{http_code}" "${package_url}")
+
+  [[ "${status}" != "200" ]]
+}
+
+function create_gitlab_workhorse_package() {
+  local archive_filename="${GITLAB_WORKHORSE_PACKAGE}"
+  local folder_to_archive="${GITLAB_WORKHORSE_FOLDER}"
+  local workhorse_folder_path="${TMP_TEST_GITLAB_WORKHORSE_PATH}"
+  local tar_working_folder="${TMP_TEST_FOLDER}"
+
+  echoinfo "Running 'tar -czvf ${archive_filename} -C ${tar_working_folder} ${folder_to_archive}'"
+  tar -czvf ${archive_filename} -C ${tar_working_folder} ${folder_to_archive}
+  du -h ${archive_filename}
+}
+
+function extract_gitlab_workhorse_package() {
+  local tar_working_folder="${TMP_TEST_FOLDER}"
+
+  echoinfo "Extracting archive to ${tar_working_folder}"
+
+  tar -xzv -C ${tar_working_folder} < /dev/stdin
+}
+
+function upload_gitlab_workhorse_package() {
+  local archive_filename="${GITLAB_WORKHORSE_PACKAGE}"
+  local package_url="${GITLAB_WORKHORSE_PACKAGE_URL}"
+  local token_header="${CURL_TOKEN_HEADER}"
+  local token="${CI_JOB_TOKEN}"
+
+  echoinfo "Uploading ${archive_filename} to ${package_url} ..."
+  curl --fail --silent --retry 3 --header "${token_header}: ${token}" --upload-file "${archive_filename}" "${package_url}"
+}
+
+function read_curl_gitlab_workhorse_package() {
+  local package_url="${GITLAB_WORKHORSE_PACKAGE_URL}"
+  local token_header="${CURL_TOKEN_HEADER}"
+  local token="${CI_JOB_TOKEN}"
+
+  echoinfo "Downloading from ${package_url} ..."
+
+  curl --fail --silent --retry 3 --header "${token_header}: ${token}" "${package_url}"
+}
+
+function download_and_extract_gitlab_workhorse_package() {
+  read_curl_gitlab_workhorse_package | extract_gitlab_workhorse_package
+}
+
+function select_gitlab_workhorse_essentials() {
+  local tmp_path="${CI_PROJECT_DIR}/tmp/${GITLAB_WORKHORSE_FOLDER}"
+  local original_gitlab_workhorse_path="${TMP_TEST_GITLAB_WORKHORSE_PATH}"
+
+  mkdir -p ${tmp_path}
+  cd ${original_gitlab_workhorse_path} && mv ${GITLAB_WORKHORSE_PACKAGE_FILES_LIST} ${tmp_path} && cd -
+  rm -rf ${original_gitlab_workhorse_path}
+
+  # Move the temp folder to its final destination
+  mv ${tmp_path} ${TMP_TEST_FOLDER}
+}

scripts/utils.sh

@@ -83,7 +83,7 @@ function install_junit_merge_gem() {
 
 function run_timed_command() {
   local cmd="${1}"
-  local metric_name="${2}"
+  local metric_name="${2:-no}"
   local timed_metric_file
   local start=$(date +%s)
 
@@ -97,7 +97,7 @@ function run_timed_command() {
   if [[ $ret -eq 0 ]]; then
     echosuccess "==> '${cmd}' succeeded in ${runtime} seconds."
 
-    if [[ -n "${metric_name}" ]]; then
+    if [[ "${metric_name}" != "no" ]]; then
       timed_metric_file=$(timed_metric_file $metric_name)
       echo "# TYPE ${metric_name} gauge" > "${timed_metric_file}"
       echo "# UNIT ${metric_name} seconds" >> "${timed_metric_file}"
@@ -132,9 +132,9 @@ function timed_metric_file() {
 }
 
 function echoerr() {
-  local header="${2}"
+  local header="${2:-no}"
 
-  if [ -n "${header}" ]; then
+  if [ "${header}" != "no" ]; then
     printf "\n\033[0;31m** %s **\n\033[0m" "${1}" >&2;
   else
     printf "\033[0;31m%s\n\033[0m" "${1}" >&2;
@@ -142,9 +142,9 @@ function echoerr() {
 }
 
 function echoinfo() {
-  local header="${2}"
+  local header="${2:-no}"
 
-  if [ -n "${header}" ]; then
+  if [ "${header}" != "no" ]; then
     printf "\n\033[0;33m** %s **\n\033[0m" "${1}" >&2;
   else
     printf "\033[0;33m%s\n\033[0m" "${1}" >&2;
@@ -152,9 +152,9 @@ function echoinfo() {
 }
 
 function echosuccess() {
-  local header="${2}"
+  local header="${2:-no}"
 
-  if [ -n "${header}" ]; then
+  if [ "${header}" != "no" ]; then
     printf "\n\033[0;32m** %s **\n\033[0m" "${1}" >&2;
   else
     printf "\033[0;32m%s\n\033[0m" "${1}" >&2;