Merge branch 'improve-review-apps-jobs-structure' into 'master'

Organize better Review Apps and QA jobs

See merge request gitlab-org/gitlab-ce!27509

.gitlab-ci.yml

@@ -28,6 +28,8 @@ stages:
   - prepare
   - merge
   - test
+  - review
+  - qa
   - post-test
   - pages
   - post-cleanup

.gitlab/ci/cng.gitlab-ci.yml

@@ -9,7 +9,7 @@ cloud-native-image:
   cache: {}
   when: manual
   script:
-    - gem install gitlab --no-document
+    - install_gitlab_gem
     - CNG_PROJECT_PATH="gitlab-org/build/CNG" BUILD_TRIGGER_TOKEN=$CI_JOB_TOKEN ./scripts/trigger-build cng
   only:
     - tags@gitlab-org/gitlab-ce

.gitlab/ci/frontend.gitlab-ci.yml

@@ -38,6 +38,10 @@ gitlab:assets:compile:
     - bundle exec rake gitlab:assets:compile
     - time scripts/build_assets_image
     - scripts/clean-old-cached-assets
+    # Play dependent manual jobs
+    - install_api_client_dependencies_with_apt
+    - play_job "review-build-cng" || true # this job might not exist so ignore the failure if it cannot be played
+    - play_job "schedule:review-build-cng" || true # this job might not exist so ignore the failure if it cannot be played
   artifacts:
     name: webpack-report
     expire_in: 31d

.gitlab/ci/qa.gitlab-ci.yml

 package-and-qa:
   image: ruby:2.5-alpine
-  stage: test
+  stage: qa
+  when: manual
   before_script: []
   dependencies: []
   cache: {}
   variables:
     GIT_DEPTH: "1"
-    API_TOKEN: "${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}"
   retry: 0
   script:
-    - apk add --update openssl curl jq
-    - gem install gitlab --no-document
-    - source ./scripts/review_apps/review-apps.sh
-    - wait_for_job_to_be_done "gitlab:assets:compile"
+    - source scripts/utils.sh
+    - install_gitlab_gem
     - ./scripts/trigger-build omnibus
-  when: manual
   only:
     - /.+/@gitlab-org/gitlab-ce
     - /.+/@gitlab-org/gitlab-ee

.gitlab/ci/review.gitlab-ci.yml

@@ -26,12 +26,10 @@
   extends: .dedicated-runner
   <<: *review-only
   image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
-  stage: test
   cache: {}
   dependencies: []
-  environment: &review-environment
-    name: review/${CI_COMMIT_REF_NAME}
-    url: https://gitlab-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}
+  before_script:
+    - source scripts/utils.sh
 
 .review-docker: &review-docker
   <<: *review-base
@@ -42,18 +40,13 @@
     - gitlab-org
     - docker
   variables: &review-docker-variables
-    GIT_DEPTH: "1"
     DOCKER_DRIVER: overlay2
     DOCKER_HOST: tcp://docker:2375
     LATEST_QA_IMAGE: "gitlab/${CI_PROJECT_NAME}-qa:nightly"
     QA_IMAGE: "${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab/${CI_PROJECT_NAME}-qa:${CI_COMMIT_REF_SLUG}"
-  before_script: []
 
 build-qa-image:
   <<: *review-docker
-  variables:
-    <<: *review-docker-variables
-    GIT_DEPTH: "20"
   stage: prepare
   script:
     - time docker build --cache-from ${LATEST_QA_IMAGE} --tag ${QA_IMAGE} ./qa/
@@ -63,16 +56,14 @@ build-qa-image:
 .review-build-cng-base: &review-build-cng-base
   image: ruby:2.5-alpine
   stage: test
-  before_script: []
+  when: manual
+  before_script:
+    - source scripts/utils.sh
+    - install_api_client_dependencies_with_apk
+    - install_gitlab_gem
   dependencies: []
   cache: {}
-  variables:
-    API_TOKEN: "${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}"
   script:
-    - apk add --update openssl curl jq
-    - gem install gitlab --no-document
-    - source ./scripts/review_apps/review-apps.sh
-    - wait_for_job_to_be_done "gitlab:assets:compile"
     - BUILD_TRIGGER_TOKEN=$REVIEW_APPS_BUILD_TRIGGER_TOKEN ./scripts/trigger-build cng
 
 review-build-cng:
@@ -85,26 +76,32 @@ schedule:review-build-cng:
 
 .review-deploy-base: &review-deploy-base
   <<: *review-base
+  stage: review
   retry: 2
   allow_failure: true
   variables:
     HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}"
     DOMAIN: "-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}"
     GITLAB_HELM_CHART_REF: "master"
-    API_TOKEN: "${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}"
-  environment:
-    <<: *review-environment
+  environment: &review-environment
+    name: review/${CI_COMMIT_REF_NAME}
+    url: https://gitlab-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}
     on_stop: review-stop
   before_script:
     - export GITLAB_SHELL_VERSION=$(<GITLAB_SHELL_VERSION)
     - export GITALY_VERSION=$(<GITALY_SERVER_VERSION)
     - export GITLAB_WORKHORSE_VERSION=$(<GITLAB_WORKHORSE_VERSION)
-    - apk update && apk add jq
-    - gem install gitlab --no-document
-    - source ./scripts/review_apps/review-apps.sh
+    - echo "${CI_ENVIRONMENT_URL}" > review_app_url.txt
+    - source scripts/utils.sh
+    - install_api_client_dependencies_with_apk
+    - source scripts/review_apps/review-apps.sh
   script:
-    - wait_for_job_to_be_done "review-build-cng"
     - perform_review_app_deployment
+  artifacts:
+    paths:
+      - review_app_url.txt
+    expire_in: 2 days
+    when: always
 
 review-deploy:
   <<: *review-deploy-base
@@ -113,15 +110,29 @@ schedule:review-deploy:
   <<: *review-deploy-base
   <<: *review-schedules-only
   script:
-    - wait_for_job_to_be_done "schedule:review-build-cng"
     - perform_review_app_deployment
 
+review-stop:
+  <<: *review-base
+  stage: review
+  when: manual
+  allow_failure: true
+  variables:
+    GIT_DEPTH: "1"
+  environment:
+    <<: *review-environment
+    action: stop
+  script:
+    - source scripts/review_apps/review-apps.sh
+    - delete
+    - cleanup
+
 .review-qa-base: &review-qa-base
   <<: *review-docker
+  stage: qa
   allow_failure: true
   variables:
     <<: *review-docker-variables
-    API_TOKEN: "${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}"
     QA_ARTIFACTS_DIR: "${CI_PROJECT_DIR}/qa"
     QA_CAN_TEST_GIT_PROTOCOL_V2: "false"
     GITLAB_USERNAME: "root"
@@ -131,40 +142,45 @@ schedule:review-deploy:
     GITHUB_ACCESS_TOKEN: "${REVIEW_APPS_QA_GITHUB_ACCESS_TOKEN}"
     EE_LICENSE: "${REVIEW_APPS_EE_LICENSE}"
     QA_DEBUG: "true"
+  dependencies:
+    - review-deploy
   artifacts:
     paths:
       - ./qa/gitlab-qa-run-*
     expire_in: 7 days
     when: always
   before_script:
-    - echo "${QA_IMAGE}"
+    - export CI_ENVIRONMENT_URL="$(cat review_app_url.txt)"
     - echo "${CI_ENVIRONMENT_URL}"
-    - apk update && apk add curl jq
-    - source ./scripts/review_apps/review-apps.sh
+    - echo "${QA_IMAGE}"
+    - source scripts/utils.sh
+    - install_api_client_dependencies_with_apk
     - gem install gitlab-qa --no-document ${GITLAB_QA_VERSION:+ --version ${GITLAB_QA_VERSION}}
 
 review-qa-smoke:
   <<: *review-qa-base
   retry: 2
   script:
-    - wait_for_job_to_be_done "review-deploy"
     - gitlab-qa Test::Instance::Smoke "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}"
 
 review-qa-all:
   <<: *review-qa-base
+  when: manual
   script:
-    - wait_for_job_to_be_done "review-deploy"
     - gitlab-qa Test::Instance::Any "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}"
-  when: manual
 
 .review-performance-base: &review-performance-base
   <<: *review-qa-base
-  script:
-    - wait_for_job_to_be_done "review-deploy"
+  stage: qa
+  before_script:
+    - export CI_ENVIRONMENT_URL="$(cat review_app_url.txt)"
+    - echo "${CI_ENVIRONMENT_URL}"
     - mkdir -p gitlab-exporter
     - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/master/index.js
-    - mkdir sitespeed-results
-    - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL"
+    - mkdir -p sitespeed-results
+  script:
+    - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "${CI_ENVIRONMENT_URL}"
+  after_script:
     - mv sitespeed-results/data/performance.json performance.json
   artifacts:
     paths:
@@ -175,42 +191,27 @@ review-qa-all:
 review-performance:
   <<: *review-performance-base
 
-review-stop:
-  <<: *review-base
-  extends: .single-script-job-dedicated-runner
-  image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
-  allow_failure: true
-  variables:
-    SCRIPT_NAME: "review_apps/review-apps.sh"
-  when: manual
-  environment:
-    <<: *review-environment
-    action: stop
-  script:
-    - source $(basename "${SCRIPT_NAME}")
-    - delete
-    - cleanup
+schedule:review-performance:
+  <<: *review-performance-base
+  <<: *review-schedules-only
+  dependencies:
+    - schedule:review-deploy
 
 schedule:review-cleanup:
   <<: *review-base
   <<: *review-schedules-only
-  stage: build
+  stage: review
   allow_failure: true
   variables:
     GIT_DEPTH: "1"
   environment:
     name: review/auto-cleanup
   before_script:
-    - gem install gitlab --no-document
+    - source scripts/utils.sh
+    - install_gitlab_gem
   script:
     - ruby -rrubygems scripts/review_apps/automated_cleanup.rb
 
-schedule:review-performance:
-  <<: *review-performance-base
-  <<: *review-schedules-only
-  script:
-    - wait_for_job_to_be_done "schedule:review-deploy"
-
 danger-review:
   extends: .dedicated-pull-cache-job
   image: registry.gitlab.com/gitlab-org/gitlab-build-images:danger

doc/development/testing_guide/review_apps.md

@@ -6,7 +6,7 @@ Review Apps are automatically deployed by each pipeline, both in
 
 ## How does it work?
 
-### CD/CD architecture diagram
+### CI/CD architecture diagram
 
 ![Review Apps CI/CD architecture](img/review_apps_cicd_architecture.png)
 
@@ -14,23 +14,29 @@ Review Apps are automatically deployed by each pipeline, both in
 <summary>Show mermaid source</summary>
 <pre>
 graph TD
-    B1 -.->|2. once gitlab:assets:compile is done,<br />triggers a CNG-mirror pipeline and wait for it to be done| A2
-    C1 -.->|2. once review-build-cng is done,<br />Helm deploys the Review App using the Cloud<br/>Native images built by the CNG-mirror pipeline| A3
-
-subgraph gitlab-ce/ee `test` stage
-    A1[gitlab:assets:compile]
-    B1[review-build-cng] -->|1. wait for| A1
-    C1[review-deploy] -->|1. wait for| B1
-    D1[review-qa-smoke] -->|1. wait for| C1
-    D1[review-qa-smoke] -.->|2. once review-deploy is done| E1>gitlab-qa runs the smoke<br/>suite against the Review App]
+    build-qa-image -.->|once the `prepare` stage is done| gitlab:assets:compile
+    review-build-cng -->|triggers a CNG-mirror pipeline and wait for it to be done| CNG-mirror
+    review-build-cng -.->|once the `test` stage is done| review-deploy
+    review-deploy -.->|once the `review` stage is done| review-qa-smoke
+
+subgraph 1. gitlab-ce/ee `prepare` stage
+    build-qa-image
     end
 
-subgraph CNG-mirror pipeline
-    A2>Cloud Native images are built];
+subgraph 2. gitlab-ce/ee `test` stage
+    gitlab:assets:compile -->|plays dependent job once done| review-build-cng
     end
 
-subgraph GCP `gitlab-review-apps` project
-    A3>"Cloud Native images are deployed to the<br />`review-apps-ce` or `review-apps-ee` Kubernetes (GKE) cluster"];
+subgraph 3. gitlab-ce/ee `review` stage
+    review-deploy["review-deploy<br /><br />Helm deploys the Review App using the Cloud<br/>Native images built by the CNG-mirror pipeline.<br /><br />Cloud Native images are deployed to the `review-apps-ce` or `review-apps-ee`<br />Kubernetes (GKE) cluster, in the GCP `gitlab-review-apps` project."]
+    end
+
+subgraph 4. gitlab-ce/ee `qa` stage
+    review-qa-smoke[review-qa-smoke<br /><br />gitlab-qa runs the smoke suite against the Review App.]
+    end
+
+subgraph CNG-mirror pipeline
+    CNG-mirror>Cloud Native images are built];
     end
 </pre>
 </details>
@@ -38,27 +44,24 @@ subgraph GCP `gitlab-review-apps` project
 ### Detailed explanation
 
 1. On every [pipeline][gitlab-pipeline] during the `test` stage, the
-  [`review-build-cng`][review-build-cng] and
-  [`review-deploy`][review-deploy] jobs are automatically started.
-    - The [`review-deploy`][review-deploy] job waits for the
-      [`review-build-cng`][review-build-cng] job to finish.
-    - The [`review-build-cng`][review-build-cng] job waits for the
-      [`gitlab:assets:compile`][gitlab:assets:compile] job to finish since the
-      [`CNG-mirror`][cng-mirror] pipeline triggered in the following step depends on it.
-1. Once the [`gitlab:assets:compile`][gitlab:assets:compile] job is done,
-  [`review-build-cng`][review-build-cng] [triggers a pipeline][cng-pipeline]
+   [`gitlab:assets:compile`][gitlab:assets:compile] job is automatically started.
+   - Once it's done, it starts the [`review-build-cng`][review-build-cng]
+     manual job since the [`CNG-mirror`][cng-mirror] pipeline triggered in the
+     following step depends on it.
+1. The [`review-build-cng`][review-build-cng] job [triggers a pipeline][cng-mirror-pipeline]
    in the [`CNG-mirror`][cng-mirror] project.
-    - The [`CNG-mirror`][cng-pipeline] pipeline creates the Docker images of
+   - The [`CNG-mirror`][cng-mirror-pipeline] pipeline creates the Docker images of
      each component (e.g. `gitlab-rails-ee`, `gitlab-shell`, `gitaly` etc.)
-      based on the commit from the [GitLab pipeline][gitlab-pipeline] and store
+     based on the commit from the [GitLab pipeline][gitlab-pipeline] and stores
      them in its [registry][cng-mirror-registry].
    - We use the [`CNG-mirror`][cng-mirror] project so that the `CNG`, (**C**loud
      **N**ative **G**itLab), project's registry is not overloaded with a
      lot of transient Docker images.
-1. Once the [`review-build-cng`][review-build-cng] job is done, the
-  [`review-deploy`][review-deploy] job deploys the Review App using
-  [the official GitLab Helm chart][helm-chart] to the
-  [`review-apps-ce`][review-apps-ce] / [`review-apps-ee`][review-apps-ee]
+   - Note that the official CNG images are built by the `cloud-native-image`
+     job, which runs only for tags, and triggers itself a [`CNG`][cng] pipeline.
+1. Once the `test` stage is done, the [`review-deploy`][review-deploy] job
+   deploys the Review App using [the official GitLab Helm chart][helm-chart] to
+   the [`review-apps-ce`][review-apps-ce] / [`review-apps-ee`][review-apps-ee]
    Kubernetes cluster on GCP.
    - The actual scripts used to deploy the Review App can be found at
      [`scripts/review_apps/review-apps.sh`][review-apps.sh].
@@ -82,71 +85,69 @@ subgraph GCP `gitlab-review-apps` project
 - If the Review App deployment fails, you can simply retry it (there's no need
   to run the [`review-stop`][gitlab-ci-yml] job first).
 - The manual [`review-stop`][gitlab-ci-yml] in the `test` stage can be used to
-  stop a Review App manually, and is also started by GitLab once a branch is
-  deleted.
-- Review Apps are cleaned up regularly using a pipeline schedule that runs
+  stop a Review App manually, and is also started by GitLab once a merge
+  request's branch is deleted after being merged.
+- Review Apps are cleaned up regularly via a pipeline schedule that runs
   the [`schedule:review-cleanup`][gitlab-ci-yml] job.
 
 ## QA runs
 
-On every [pipeline][gitlab-pipeline] during the `test` stage, the
-`review-qa-smoke` job is automatically started: it runs the QA smoke suite.
-You can also manually start the `review-qa-all`: it runs the QA full suite.
+On every [pipeline][gitlab-pipeline] in the `qa` stage (which comes after the
+`review` stage), the `review-qa-smoke` job is automatically started and it runs
+the QA smoke suite.
 
-Note that both jobs first wait for the `review-deploy` job to be finished.
+You can also manually start the `review-qa-all`: it runs the full QA suite.
 
 ## Performance Metrics
 
-On every [pipeline][gitlab-pipeline] during the `test` stage, the
+On every [pipeline][gitlab-pipeline] in the `qa` stage, the
 `review-performance` job is automatically started: this job does basic
-browser performance testing using [Sitespeed.io Container](https://docs.gitlab.com/ee/user/project/merge_requests/browser_performance_testing.html) .
+browser performance testing using a
+[Sitespeed.io Container](https://docs.gitlab.com/ee/user/project/merge_requests/browser_performance_testing.html).
 
-This job waits for the `review-deploy` job to be finished.
+## How to:
 
-## How to?
-
-### Log into my Review App?
+### Log into my Review App
 
 The default username is `root` and its password can be found in the 1Password
 secure note named **gitlab-{ce,ee} Review App's root password**.
 
-### Enable a feature flag for my Review App?
+### Enable a feature flag for my Review App
 
 1. Open your Review App and log in as documented above.
 1. Create a personal access token.
 1. Enable the feature flag using the [Feature flag API](../../api/features.md).
 
-### Find my Review App slug?
+### Find my Review App slug
 
 1. Open the `review-deploy` job.
 1. Look for `Checking for previous deployment of review-*`.
 1. For instance for `Checking for previous deployment of review-qa-raise-e-12chm0`,
    your Review App slug would be `review-qa-raise-e-12chm0` in this case.
 
-### Run a Rails console?
+### Run a Rails console
 
 1. [Filter Workloads by your Review App slug](https://console.cloud.google.com/kubernetes/workload?project=gitlab-review-apps)
-  , e.g. `review-29951-issu-id2qax`.
-1. Find and open the `task-runner` Deployment, e.g. `review-29951-issu-id2qax-task-runner`.
-1. Click on the Pod in the "Managed pods" section, e.g. `review-29951-issu-id2qax-task-runner-d5455cc8-2lsvz`.
+  , e.g. `review-qa-raise-e-12chm0`.
+1. Find and open the `task-runner` Deployment, e.g. `review-qa-raise-e-12chm0-task-runner`.
+1. Click on the Pod in the "Managed pods" section, e.g. `review-qa-raise-e-12chm0-task-runner-d5455cc8-2lsvz`.
 1. Click on the `KUBECTL` dropdown, then `Exec` -> `task-runner`.
 1. Replace `-c task-runner -- ls` with `-it -- gitlab-rails console` from the
    default command or
-  - Run `kubectl exec --namespace review-apps-ce review-29951-issu-id2qax-task-runner-d5455cc8-2lsvz -it -- gitlab-rails console`
-    and
+   - Run `kubectl exec --namespace review-apps-ce review-qa-raise-e-12chm0-task-runner-d5455cc8-2lsvz -it -- gitlab-rails console` and
      - Replace `review-apps-ce` with `review-apps-ee` if the Review App
        is running EE, and
-  - Replace `review-29951-issu-id2qax-task-runner-d5455cc8-2lsvz`
+     - Replace `review-qa-raise-e-12chm0-task-runner-d5455cc8-2lsvz`
        with your Pod's name.
 
-### Dig into a Pod's logs?
+### Dig into a Pod's logs
 
-1. [Filter Workloads by your Review App slug](https://console.cloud.google.com/kubernetes/workload?project=gitlab-review-apps)
-  , e.g. `review-1979-1-mul-dnvlhv`.
+1. [Filter Workloads by your Review App slug](https://console.cloud.google.com/kubernetes/workload?project=gitlab-review-apps),
+   e.g. `review-qa-raise-e-12chm0`.
 1. Find and open the `migrations` Deployment, e.g.
-  `review-1979-1-mul-dnvlhv-migrations.1`.
+   `review-qa-raise-e-12chm0-migrations.1`.
 1. Click on the Pod in the "Managed pods" section, e.g.
-  `review-1979-1-mul-dnvlhv-migrations.1-nqwtx`.
+   `review-qa-raise-e-12chm0-migrations.1-nqwtx`.
 1. Click on the `Container logs` link.
 
 ## Frequently Asked Questions
@@ -182,7 +183,8 @@ find a way to limit it to only us.**
 [review-build-cng]: https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/149511623
 [review-deploy]: https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/149511624
 [cng-mirror]: https://gitlab.com/gitlab-org/build/CNG-mirror
-[cng-pipeline]: https://gitlab.com/gitlab-org/build/CNG-mirror/pipelines/44364657
+[cng]: https://gitlab.com/gitlab-org/build/CNG
+[cng-mirror-pipeline]: https://gitlab.com/gitlab-org/build/CNG-mirror/pipelines/44364657
 [cng-mirror-registry]: https://gitlab.com/gitlab-org/build/CNG-mirror/container_registry
 [helm-chart]: https://gitlab.com/charts/gitlab/
 [review-apps-ce]: https://console.cloud.google.com/kubernetes/clusters/details/us-central1-a/review-apps-ce?project=gitlab-review-apps

scripts/review_apps/review-apps.sh

 [[ "$TRACE" ]] && set -x
 export TILLER_NAMESPACE="$KUBE_NAMESPACE"
 
-function echoerr() {
-  local header="${2}"
-
-  if [ -n "${header}" ]; then
-    printf "\n\033[0;31m** %s **\n\033[0m" "${1}" >&2;
-  else
-    printf "\033[0;31m%s\n\033[0m" "${1}" >&2;
-  fi
-}
-
-function echoinfo() {
-  local header="${2}"
-
-  if [ -n "${header}" ]; then
-    printf "\n\033[0;33m** %s **\n\033[0m" "${1}" >&2;
-  else
-    printf "\033[0;33m%s\n\033[0m" "${1}" >&2;
-  fi
-}
-
 function deployExists() {
   local namespace="${1}"
   local deploy="${2}"
@@ -328,80 +308,3 @@ function add_license() {
     puts "License added";
     '
 }
-
-function get_job_id() {
-  local job_name="${1}"
-  local query_string="${2:+&${2}}"
-
-  local max_page=3
-  local page=1
-
-  while true; do
-    local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/jobs?per_page=100&page=${page}${query_string}"
-    echoinfo "GET ${url}"
-
-    local job_id
-    job_id=$(curl --silent --show-error --header "PRIVATE-TOKEN: ${API_TOKEN}" "${url}" | jq "map(select(.name == \"${job_name}\")) | map(.id) | last")
-    [[ "${job_id}" == "null" && "${page}" -lt "$max_page" ]] || break
-
-    let "page++"
-  done
-
-  if [[ "${job_id}" == "" ]]; then
-    echoerr "The '${job_name}' job ID couldn't be retrieved!"
-  else
-    echoinfo "The '${job_name}' job ID is ${job_id}"
-    echo "${job_id}"
-  fi
-}
-
-function play_job() {
-  local job_name="${1}"
-  local job_id
-  job_id=$(get_job_id "${job_name}" "scope=manual");
-  if [ -z "${job_id}" ]; then return; fi
-
-  local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/jobs/${job_id}/play"
-  echoinfo "POST ${url}"
-
-  local job_url
-  job_url=$(curl --silent --show-error --request POST --header "PRIVATE-TOKEN: ${API_TOKEN}" "${url}" | jq ".web_url")
-  echoinfo "Manual job '${job_name}' started at: ${job_url}"
-}
-
-function wait_for_job_to_be_done() {
-  local job_name="${1}"
-  local query_string="${2}"
-  local job_id
-  job_id=$(get_job_id "${job_name}" "${query_string}")
-  if [ -z "${job_id}" ]; then return; fi
-
-  echoinfo "Waiting for the '${job_name}' job to finish..."
-
-  local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/jobs/${job_id}"
-  echoinfo "GET ${url}"
-
-  # In case the job hasn't finished yet. Keep trying until the job times out.
-  local interval=30
-  local elapsed_seconds=0
-  while true; do
-    local job_status
-    job_status=$(curl --silent --show-error --header "PRIVATE-TOKEN: ${API_TOKEN}" "${url}" | jq ".status" | sed -e s/\"//g)
-    [[ "${job_status}" == "pending" || "${job_status}" == "running" ]] || break
-
-    printf "."
-    let "elapsed_seconds+=interval"
-    sleep ${interval}
-  done
-
-  local elapsed_minutes=$((elapsed_seconds / 60))
-  echoinfo "Waited '${job_name}' for ${elapsed_minutes} minutes."
-
-  if [[ "${job_status}" == "failed" ]]; then
-    echoerr "The '${job_name}' failed."
-  elif [[ "${job_status}" == "manual" ]]; then
-    echoinfo "The '${job_name}' is manual."
-  else
-    echoinfo "The '${job_name}' passed."
-  fi
-}

scripts/utils.sh

-retry() {
+function retry() {
     if eval "$@"; then
         return 0
     fi
@@ -13,15 +13,15 @@ retry() {
     return 1
 }
 
-setup_db_user_only() {
+function setup_db_user_only() {
     if [ "$GITLAB_DATABASE" = "postgresql" ]; then
-        . scripts/create_postgres_user.sh
+        source scripts/create_postgres_user.sh
     else
-        . scripts/create_mysql_user.sh
+        source scripts/create_mysql_user.sh
     fi
 }
 
-setup_db() {
+function setup_db() {
     setup_db_user_only
 
     bundle exec rake db:drop db:create db:schema:load db:migrate
@@ -30,3 +30,129 @@ setup_db() {
         bundle exec rake add_limits_mysql
     fi
 }
+
+function install_api_client_dependencies_with_apk() {
+  apk add --update openssl curl jq
+}
+
+function install_api_client_dependencies_with_apt() {
+  apt update && apt install jq -y
+}
+
+function install_gitlab_gem() {
+  gem install gitlab --no-document
+}
+
+function echoerr() {
+  local header="${2}"
+
+  if [ -n "${header}" ]; then
+    printf "\n\033[0;31m** %s **\n\033[0m" "${1}" >&2;
+  else
+    printf "\033[0;31m%s\n\033[0m" "${1}" >&2;
+  fi
+}
+
+function echoinfo() {
+  local header="${2}"
+
+  if [ -n "${header}" ]; then
+    printf "\n\033[0;33m** %s **\n\033[0m" "${1}" >&2;
+  else
+    printf "\033[0;33m%s\n\033[0m" "${1}" >&2;
+  fi
+}
+
+function get_job_id() {
+  local job_name="${1}"
+  local query_string="${2:+&${2}}"
+  local api_token="${API_TOKEN-${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}}"
+  if [ -z "${api_token}" ]; then
+    echoerr "Please provide an API token with \$API_TOKEN or \$GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN."
+    return
+  fi
+
+  local max_page=3
+  local page=1
+
+  while true; do
+    local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/jobs?per_page=100&page=${page}${query_string}"
+    echoinfo "GET ${url}"
+
+    local job_id
+    job_id=$(curl --silent --show-error --header "PRIVATE-TOKEN: ${api_token}" "${url}" | jq "map(select(.name == \"${job_name}\")) | map(.id) | last")
+    [[ "${job_id}" == "null" && "${page}" -lt "$max_page" ]] || break
+
+    let "page++"
+  done
+
+  if [[ "${job_id}" == "" ]]; then
+    echoerr "The '${job_name}' job ID couldn't be retrieved!"
+  else
+    echoinfo "The '${job_name}' job ID is ${job_id}"
+    echo "${job_id}"
+  fi
+}
+
+function play_job() {
+  local job_name="${1}"
+  local job_id
+  job_id=$(get_job_id "${job_name}" "scope=manual");
+  if [ -z "${job_id}" ]; then return; fi
+
+  local api_token="${API_TOKEN-${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}}"
+  if [ -z "${api_token}" ]; then
+    echoerr "Please provide an API token with \$API_TOKEN or \$GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN."
+    return
+  fi
+
+  local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/jobs/${job_id}/play"
+  echoinfo "POST ${url}"
+
+  local job_url
+  job_url=$(curl --silent --show-error --request POST --header "PRIVATE-TOKEN: ${api_token}" "${url}" | jq ".web_url")
+  echoinfo "Manual job '${job_name}' started at: ${job_url}"
+}
+
+function wait_for_job_to_be_done() {
+  local job_name="${1}"
+  local query_string="${2}"
+  local job_id
+  job_id=$(get_job_id "${job_name}" "${query_string}")
+  if [ -z "${job_id}" ]; then return; fi
+
+  local api_token="${API_TOKEN-${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}}"
+  if [ -z "${api_token}" ]; then
+    echoerr "Please provide an API token with \$API_TOKEN or \$GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN."
+    return
+  fi
+
+  echoinfo "Waiting for the '${job_name}' job to finish..."
+
+  local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/jobs/${job_id}"
+  echoinfo "GET ${url}"
+
+  # In case the job hasn't finished yet. Keep trying until the job times out.
+  local interval=30
+  local elapsed_seconds=0
+  while true; do
+    local job_status
+    job_status=$(curl --silent --show-error --header "PRIVATE-TOKEN: ${api_token}" "${url}" | jq ".status" | sed -e s/\"//g)
+    [[ "${job_status}" == "pending" || "${job_status}" == "running" ]] || break
+
+    printf "."
+    let "elapsed_seconds+=interval"
+    sleep ${interval}
+  done
+
+  local elapsed_minutes=$((elapsed_seconds / 60))
+  echoinfo "Waited '${job_name}' for ${elapsed_minutes} minutes."
+
+  if [[ "${job_status}" == "failed" ]]; then
+    echoerr "The '${job_name}' failed."
+  elif [[ "${job_status}" == "manual" ]]; then
+    echoinfo "The '${job_name}' is manual."
+  else
+    echoinfo "The '${job_name}' passed."
+  fi
+}