Commit 20702d80 authored by Rémy Coutable's avatar Rémy Coutable

Merge branch 'ci-predefined-variables' into 'master'

Add predefined CI variables to GitLab

## What does this MR do?
This adds predefined CI variables to GitLab for container registry, pipelines, project name, etc. It also makes sure that all currently documented variables are send from GitLab. This is added to follow up on this proposal: https://gitlab.com/gitlab-org/gitlab-ci-multi-runner/issues/185#note_11844286. To migrate almost all variables out of Runner to GitLab to simplify adding a new of variables without the need for changing the GitLab Runner.

## Why was this MR needed?
Our CI variables miss a lot of crucial information that should be easily accessible. This tries to fill this gap.

## What are the relevant issue numbers?
Fixes https://gitlab.com/gitlab-org/gitlab-ci-multi-runner/issues/185, https://gitlab.com/gitlab-org/gitlab-ce/issues/18164, https://gitlab.com/gitlab-org/gitlab-ce/issues/18075.

## Does this MR meet the acceptance criteria?

- [ ] [CHANGELOG](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CHANGELOG) entry added
- [ ] [Documentation created/updated](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/development/doc_styleguide.md)
- [ ] API support added
- Tests
  - [ ] Added for this feature/bug
  - [ ] All builds are passing
- [ ] Conform by the [style guides](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CONTRIBUTING.md#style-guides)
- [ ] Branch has no merge conflicts with `master` (if you do - rebase it please)
- [ ] [Squashed related commits together](https://git-scm.com/book/en/Git-Tools-Rewriting-History#Squashing-Commits)


See merge request !4826
parent 386e8c08
...@@ -23,6 +23,7 @@ v 8.10.0 (unreleased) ...@@ -23,6 +23,7 @@ v 8.10.0 (unreleased)
- Escape file extension when parsing search results !5141 (winniehell) - Escape file extension when parsing search results !5141 (winniehell)
- Apply the trusted_proxies config to the rack request object for use with rack_attack - Apply the trusted_proxies config to the rack request object for use with rack_attack
- Upgrade to Rails 4.2.7. !5236 - Upgrade to Rails 4.2.7. !5236
- Extend exposed environment variables for CI builds
- Allow to pull code with deploy key from public projects - Allow to pull code with deploy key from public projects
- Add Sidekiq queue duration to transaction metrics. - Add Sidekiq queue duration to transaction metrics.
- Add a new column `artifacts_size` to table `ci_builds` !4964 - Add a new column `artifacts_size` to table `ci_builds` !4964
......
...@@ -145,7 +145,15 @@ module Ci ...@@ -145,7 +145,15 @@ module Ci
end end
def variables def variables
predefined_variables + yaml_variables + project_variables + trigger_variables variables = predefined_variables
variables += project.predefined_variables
variables += pipeline.predefined_variables
variables += runner.predefined_variables if runner
variables += project.container_registry_variables
variables += yaml_variables
variables += project.secret_variables
variables += trigger_request.user_variables if trigger_request
variables
end end
def merge_request def merge_request
...@@ -430,28 +438,23 @@ module Ci ...@@ -430,28 +438,23 @@ module Ci
self.update(erased_by: user, erased_at: Time.now, artifacts_expire_at: nil) self.update(erased_by: user, erased_at: Time.now, artifacts_expire_at: nil)
end end
def project_variables
project.variables.map do |variable|
{ key: variable.key, value: variable.value, public: false }
end
end
def trigger_variables
if trigger_request && trigger_request.variables
trigger_request.variables.map do |key, value|
{ key: key, value: value, public: false }
end
else
[]
end
end
def predefined_variables def predefined_variables
variables = [] variables = [
variables << { key: :CI_BUILD_TAG, value: ref, public: true } if tag? { key: 'CI', value: 'true', public: true },
variables << { key: :CI_BUILD_NAME, value: name, public: true } { key: 'GITLAB_CI', value: 'true', public: true },
variables << { key: :CI_BUILD_STAGE, value: stage, public: true } { key: 'CI_BUILD_ID', value: id.to_s, public: true },
variables << { key: :CI_BUILD_TRIGGERED, value: 'true', public: true } if trigger_request { key: 'CI_BUILD_TOKEN', value: token, public: false },
{ key: 'CI_BUILD_REF', value: sha, public: true },
{ key: 'CI_BUILD_BEFORE_SHA', value: before_sha, public: true },
{ key: 'CI_BUILD_REF_NAME', value: ref, public: true },
{ key: 'CI_BUILD_NAME', value: name, public: true },
{ key: 'CI_BUILD_STAGE', value: stage, public: true },
{ key: 'CI_SERVER_NAME', value: 'GitLab', public: true },
{ key: 'CI_SERVER_VERSION', value: Gitlab::VERSION, public: true },
{ key: 'CI_SERVER_REVISION', value: Gitlab::REVISION, public: true }
]
variables << { key: 'CI_BUILD_TAG', value: ref, public: true } if tag?
variables << { key: 'CI_BUILD_TRIGGERED', value: 'true', public: true } if trigger_request
variables variables
end end
......
...@@ -198,6 +198,12 @@ module Ci ...@@ -198,6 +198,12 @@ module Ci
Note.for_commit_id(sha) Note.for_commit_id(sha)
end end
def predefined_variables
[
{ key: 'CI_PIPELINE_ID', value: id.to_s, public: true }
]
end
private private
def build_builds_for_stages(stages, user, status, trigger_request) def build_builds_for_stages(stages, user, status, trigger_request)
......
...@@ -114,6 +114,14 @@ module Ci ...@@ -114,6 +114,14 @@ module Ci
tag_list.any? tag_list.any?
end end
def predefined_variables
[
{ key: 'CI_RUNNER_ID', value: id.to_s, public: true },
{ key: 'CI_RUNNER_DESCRIPTION', value: description, public: true },
{ key: 'CI_RUNNER_TAGS', value: tag_list.to_s, public: true }
]
end
private private
def tag_constraints def tag_constraints
......
...@@ -7,5 +7,13 @@ module Ci ...@@ -7,5 +7,13 @@ module Ci
has_many :builds, class_name: 'Ci::Build' has_many :builds, class_name: 'Ci::Build'
serialize :variables serialize :variables
def user_variables
return [] unless variables
variables.map do |key, value|
{ key: key, value: value, public: false }
end
end
end end
end end
...@@ -1180,4 +1180,34 @@ class Project < ActiveRecord::Base ...@@ -1180,4 +1180,34 @@ class Project < ActiveRecord::Base
def ensure_dir_exist def ensure_dir_exist
gitlab_shell.add_namespace(repository_storage_path, namespace.path) gitlab_shell.add_namespace(repository_storage_path, namespace.path)
end end
def predefined_variables
[
{ key: 'CI_PROJECT_ID', value: id.to_s, public: true },
{ key: 'CI_PROJECT_NAME', value: path, public: true },
{ key: 'CI_PROJECT_PATH', value: path_with_namespace, public: true },
{ key: 'CI_PROJECT_NAMESPACE', value: namespace.path, public: true },
{ key: 'CI_PROJECT_URL', value: web_url, public: true }
]
end
def container_registry_variables
return [] unless Gitlab.config.registry.enabled
variables = [
{ key: 'CI_REGISTRY', value: Gitlab.config.registry.host_port, public: true }
]
if container_registry_enabled?
variables << { key: 'CI_REGISTRY_IMAGE', value: container_registry_repository_url, public: true }
end
variables
end
def secret_variables
variables.map do |variable|
{ key: variable.key, value: variable.value, public: false }
end
end
end end
...@@ -18,25 +18,35 @@ The `API_TOKEN` will take the Secure Variable value: `SECURE`. ...@@ -18,25 +18,35 @@ The `API_TOKEN` will take the Secure Variable value: `SECURE`.
### Predefined variables (Environment Variables) ### Predefined variables (Environment Variables)
| Variable | Runner | Description | | Variable | GitLab | Runner | Description |
|-------------------------|-----|--------| |-------------------------|--------|--------|-------------|
| **CI** | 0.4 | Mark that build is executed in CI environment | | **CI** | all | 0.4 | Mark that build is executed in CI environment |
| **GITLAB_CI** | all | Mark that build is executed in GitLab CI environment | | **GITLAB_CI** | all | all | Mark that build is executed in GitLab CI environment |
| **CI_SERVER** | all | Mark that build is executed in CI environment | | **CI_SERVER** | all | all | Mark that build is executed in CI environment |
| **CI_SERVER_NAME** | all | CI server that is used to coordinate builds | | **CI_SERVER_NAME** | all | all | The name of CI server that is used to coordinate builds |
| **CI_SERVER_VERSION** | all | Not yet defined | | **CI_SERVER_VERSION** | all | all | GitLab version that is used to schedule builds |
| **CI_SERVER_REVISION** | all | Not yet defined | | **CI_SERVER_REVISION** | all | all | GitLab revision that is used to schedule builds |
| **CI_BUILD_REF** | all | The commit revision for which project is built | | **CI_BUILD_ID** | all | all | The unique id of the current build that GitLab CI uses internally |
| **CI_BUILD_TAG** | 0.5 | The commit tag name. Present only when building tags. | | **CI_BUILD_REF** | all | all | The commit revision for which project is built |
| **CI_BUILD_NAME** | 0.5 | The name of the build as defined in `.gitlab-ci.yml` | | **CI_BUILD_TAG** | all | 0.5 | The commit tag name. Present only when building tags. |
| **CI_BUILD_STAGE** | 0.5 | The name of the stage as defined in `.gitlab-ci.yml` | | **CI_BUILD_NAME** | all | 0.5 | The name of the build as defined in `.gitlab-ci.yml` |
| **CI_BUILD_REF_NAME** | all | The branch or tag name for which project is built | | **CI_BUILD_STAGE** | all | 0.5 | The name of the stage as defined in `.gitlab-ci.yml` |
| **CI_BUILD_ID** | all | The unique id of the current build that GitLab CI uses internally | | **CI_BUILD_REF_NAME** | all | all | The branch or tag name for which project is built |
| **CI_BUILD_REPO** | all | The URL to clone the Git repository | | **CI_BUILD_REPO** | all | all | The URL to clone the Git repository |
| **CI_BUILD_TRIGGERED** | 0.5 | The flag to indicate that build was [triggered] | | **CI_BUILD_TRIGGERED** | all | 0.5 | The flag to indicate that build was [triggered] |
| **CI_BUILD_TOKEN** | 1.2 | Token used for authenticating with the GitLab Container Registry | | **CI_BUILD_TOKEN** | all | 1.2 | Token used for authenticating with the GitLab Container Registry |
| **CI_PROJECT_ID** | all | The unique id of the current project that GitLab CI uses internally | | **CI_PIPELINE_ID** | 8.10 | 0.5 | The unique id of the current pipeline that GitLab CI uses internally |
| **CI_PROJECT_DIR** | all | The full path where the repository is cloned and where the build is ran | | **CI_PROJECT_ID** | all | all | The unique id of the current project that GitLab CI uses internally |
| **CI_PROJECT_NAME** | 8.10 | 0.5 | The project name that is currently being built |
| **CI_PROJECT_NAMESPACE**| 8.10 | 0.5 | The project namespace that is currently being built |
| **CI_PROJECT_PATH** | 8.10 | 0.5 | The namespace with project name |
| **CI_PROJECT_URL** | 8.10 | 0.5 | The HTTP address to access project |
| **CI_PROJECT_DIR** | all | all | The full path where the repository is cloned and where the build is run |
| **CI_REGISTRY** | 8.10 | 0.5 | If the Container Registry is enabled it returns the address of GitLab's Container Registry |
| **CI_REGISTRY_IMAGE** | 8.10 | 0.5 | If the Container Registry is enabled for the project it returnes the address of the registry tied to the specific project |
| **CI_RUNNER_ID** | 8.10 | 0.5 | The unique id of runner being used |
| **CI_RUNNER_DESCRIPTION** | 8.10 | 0.5 | The description of the runner as saved in GitLab |
| **CI_RUNNER_TAGS** | 8.10 | 0.5 | The defined runner tags |
**Some of the variables are only available when using runner with at least defined version.** **Some of the variables are only available when using runner with at least defined version.**
...@@ -46,18 +56,28 @@ Example values: ...@@ -46,18 +56,28 @@ Example values:
export CI_BUILD_ID="50" export CI_BUILD_ID="50"
export CI_BUILD_REF="1ecfd275763eff1d6b4844ea3168962458c9f27a" export CI_BUILD_REF="1ecfd275763eff1d6b4844ea3168962458c9f27a"
export CI_BUILD_REF_NAME="master" export CI_BUILD_REF_NAME="master"
export CI_BUILD_REPO="https://gitlab.com/gitlab-org/gitlab-ce.git" export CI_BUILD_REPO="https://gitab-ci-token:abcde-1234ABCD5678ef@gitlab.com/gitlab-org/gitlab-ce.git"
export CI_BUILD_TAG="1.0.0" export CI_BUILD_TAG="1.0.0"
export CI_BUILD_NAME="spec:other" export CI_BUILD_NAME="spec:other"
export CI_BUILD_STAGE="test" export CI_BUILD_STAGE="test"
export CI_BUILD_TRIGGERED="true" export CI_BUILD_TRIGGERED="true"
export CI_BUILD_TOKEN="abcde-1234ABCD5678ef" export CI_BUILD_TOKEN="abcde-1234ABCD5678ef"
export CI_PROJECT_DIR="/builds/gitlab-org/gitlab-ce" export CI_PIPELINE_ID="1000"
export CI_PROJECT_ID="34" export CI_PROJECT_ID="34"
export CI_PROJECT_DIR="/builds/gitlab-org/gitlab-ce"
export CI_PROJECT_NAME="gitlab-ce"
export CI_PROJECT_NAMESPACE="gitlab-org"
export CI_PROJECT_PATH="gitlab-org/gitlab-ce"
export CI_PROJECT_URL="https://gitlab.com/gitlab-org/gitlab-ce"
export CI_REGISTRY="registry.gitlab.com"
export CI_REGISTRY_IMAGE="registry.gitlab.com/gitlab-org/gitlab-ce"
export CI_RUNNER_ID="10"
export CI_RUNNER_DESCRIPTION="my runner"
export CI_RUNNER_TAGS="docker, linux"
export CI_SERVER="yes" export CI_SERVER="yes"
export CI_SERVER_NAME="GitLab CI" export CI_SERVER_NAME="GitLab"
export CI_SERVER_REVISION="" export CI_SERVER_REVISION="8.9.0"
export CI_SERVER_VERSION="" export CI_SERVER_VERSION="70606bf"
``` ```
### YAML-defined variables ### YAML-defined variables
......
...@@ -191,79 +191,87 @@ describe Ci::Build, models: true do ...@@ -191,79 +191,87 @@ describe Ci::Build, models: true do
end end
describe '#variables' do describe '#variables' do
let(:container_registry_enabled) { false }
let(:predefined_variables) do let(:predefined_variables) do
[ [
{ key: :CI_BUILD_NAME, value: 'test', public: true }, { key: 'CI', value: 'true', public: true },
{ key: :CI_BUILD_STAGE, value: 'test', public: true }, { key: 'GITLAB_CI', value: 'true', public: true },
{ key: 'CI_BUILD_ID', value: build.id.to_s, public: true },
{ key: 'CI_BUILD_TOKEN', value: build.token, public: false },
{ key: 'CI_BUILD_REF', value: build.sha, public: true },
{ key: 'CI_BUILD_BEFORE_SHA', value: build.before_sha, public: true },
{ key: 'CI_BUILD_REF_NAME', value: 'master', public: true },
{ key: 'CI_BUILD_NAME', value: 'test', public: true },
{ key: 'CI_BUILD_STAGE', value: 'test', public: true },
{ key: 'CI_SERVER_NAME', value: 'GitLab', public: true },
{ key: 'CI_SERVER_VERSION', value: Gitlab::VERSION, public: true },
{ key: 'CI_SERVER_REVISION', value: Gitlab::REVISION, public: true },
{ key: 'CI_PROJECT_ID', value: project.id.to_s, public: true },
{ key: 'CI_PROJECT_NAME', value: project.path, public: true },
{ key: 'CI_PROJECT_PATH', value: project.path_with_namespace, public: true },
{ key: 'CI_PROJECT_NAMESPACE', value: project.namespace.path, public: true },
{ key: 'CI_PROJECT_URL', value: project.web_url, public: true },
{ key: 'CI_PIPELINE_ID', value: pipeline.id.to_s, public: true }
] ]
end end
before do
stub_container_registry_config(enabled: container_registry_enabled, host_port: 'registry.example.com')
end
subject { build.variables } subject { build.variables }
context 'returns variables' do context 'returns variables' do
let(:yaml_variables) do
[
{ key: :DB_NAME, value: 'postgres', public: true }
]
end
before do before do
build.yaml_variables = yaml_variables build.yaml_variables = []
end end
it { is_expected.to eq(predefined_variables + yaml_variables) } it { is_expected.to eq(predefined_variables) }
end
context 'for tag' do context 'when build is for tag' do
let(:tag_variable) do let(:tag_variable) do
[ { key: 'CI_BUILD_TAG', value: 'master', public: true }
{ key: :CI_BUILD_TAG, value: 'master', public: true }
]
end end
before do before do
build.update_attributes(tag: true) build.update_attributes(tag: true)
end end
it { is_expected.to eq(tag_variable + predefined_variables + yaml_variables) } it { is_expected.to include(tag_variable) }
end end
context 'and secure variables' do context 'when secure variable is defined' do
let(:secure_variables) do let(:secure_variable) do
[
{ key: 'SECRET_KEY', value: 'secret_value', public: false } { key: 'SECRET_KEY', value: 'secret_value', public: false }
]
end end
before do before do
build.project.variables << Ci::Variable.new(key: 'SECRET_KEY', value: 'secret_value') build.project.variables << Ci::Variable.new(key: 'SECRET_KEY', value: 'secret_value')
end end
it { is_expected.to eq(predefined_variables + yaml_variables + secure_variables) } it { is_expected.to include(secure_variable) }
end
context 'and trigger variables' do context 'when build is for triggers' do
let(:trigger) { create(:ci_trigger, project: project) } let(:trigger) { create(:ci_trigger, project: project) }
let(:trigger_request) { create(:ci_trigger_request_with_variables, pipeline: pipeline, trigger: trigger) } let(:trigger_request) { create(:ci_trigger_request_with_variables, pipeline: pipeline, trigger: trigger) }
let(:trigger_variables) do let(:user_trigger_variable) do
[
{ key: :TRIGGER_KEY, value: 'TRIGGER_VALUE', public: false } { key: :TRIGGER_KEY, value: 'TRIGGER_VALUE', public: false }
]
end end
let(:predefined_trigger_variable) do let(:predefined_trigger_variable) do
[ { key: 'CI_BUILD_TRIGGERED', value: 'true', public: true }
{ key: :CI_BUILD_TRIGGERED, value: 'true', public: true }
]
end end
before do before do
build.trigger_request = trigger_request build.trigger_request = trigger_request
end end
it { is_expected.to eq(predefined_variables + predefined_trigger_variable + yaml_variables + secure_variables + trigger_variables) } it { is_expected.to include(user_trigger_variable) }
end it { is_expected.to include(predefined_trigger_variable) }
end
end end
context 'when yaml_variables is undefined' do context 'when yaml_variables are undefined' do
before do before do
build.yaml_variables = nil build.yaml_variables = nil
end end
...@@ -310,6 +318,58 @@ describe Ci::Build, models: true do ...@@ -310,6 +318,58 @@ describe Ci::Build, models: true do
end end
end end
end end
context 'when container registry is enabled' do
let(:container_registry_enabled) { true }
let(:ci_registry) do
{ key: 'CI_REGISTRY', value: 'registry.example.com', public: true }
end
let(:ci_registry_image) do
{ key: 'CI_REGISTRY_IMAGE', value: project.container_registry_repository_url, public: true }
end
context 'and is disabled for project' do
before do
project.update(container_registry_enabled: false)
end
it { is_expected.to include(ci_registry) }
it { is_expected.not_to include(ci_registry_image) }
end
context 'and is enabled for project' do
before do
project.update(container_registry_enabled: true)
end
it { is_expected.to include(ci_registry) }
it { is_expected.to include(ci_registry_image) }
end
end
context 'when runner is assigned to build' do
let(:runner) { create(:ci_runner, description: 'description', tag_list: ['docker', 'linux']) }
before do
build.update(runner: runner)
end
it { is_expected.to include({ key: 'CI_RUNNER_ID', value: runner.id.to_s, public: true }) }
it { is_expected.to include({ key: 'CI_RUNNER_DESCRIPTION', value: 'description', public: true }) }
it { is_expected.to include({ key: 'CI_RUNNER_TAGS', value: 'docker, linux', public: true }) }
end
context 'returns variables in valid order' do
before do
allow(build).to receive(:predefined_variables) { ['predefined'] }
allow(project).to receive(:predefined_variables) { ['project'] }
allow(pipeline).to receive(:predefined_variables) { ['pipeline'] }
allow(build).to receive(:yaml_variables) { ['yaml'] }
allow(project).to receive(:secret_variables) { ['secret'] }
end
it { is_expected.to eq(%w[predefined project pipeline yaml secret]) }
end
end end
describe '#has_tags?' do describe '#has_tags?' do
......
...@@ -73,12 +73,12 @@ describe Ci::API::API do ...@@ -73,12 +73,12 @@ describe Ci::API::API do
post ci_api("/builds/register"), token: runner.token, info: { platform: :darwin } post ci_api("/builds/register"), token: runner.token, info: { platform: :darwin }
expect(response).to have_http_status(201) expect(response).to have_http_status(201)
expect(json_response["variables"]).to eq([ expect(json_response["variables"]).to include(
{ "key" => "CI_BUILD_NAME", "value" => "spinach", "public" => true }, { "key" => "CI_BUILD_NAME", "value" => "spinach", "public" => true },
{ "key" => "CI_BUILD_STAGE", "value" => "test", "public" => true }, { "key" => "CI_BUILD_STAGE", "value" => "test", "public" => true },
{ "key" => "DB_NAME", "value" => "postgres", "public" => true }, { "key" => "DB_NAME", "value" => "postgres", "public" => true },
{ "key" => "SECRET_KEY", "value" => "secret_value", "public" => false } { "key" => "SECRET_KEY", "value" => "secret_value", "public" => false }
]) )
end end
it "returns variables for triggers" do it "returns variables for triggers" do
...@@ -92,14 +92,14 @@ describe Ci::API::API do ...@@ -92,14 +92,14 @@ describe Ci::API::API do
post ci_api("/builds/register"), token: runner.token, info: { platform: :darwin } post ci_api("/builds/register"), token: runner.token, info: { platform: :darwin }
expect(response).to have_http_status(201) expect(response).to have_http_status(201)
expect(json_response["variables"]).to eq([ expect(json_response["variables"]).to include(
{ "key" => "CI_BUILD_NAME", "value" => "spinach", "public" => true }, { "key" => "CI_BUILD_NAME", "value" => "spinach", "public" => true },
{ "key" => "CI_BUILD_STAGE", "value" => "test", "public" => true }, { "key" => "CI_BUILD_STAGE", "value" => "test", "public" => true },
{ "key" => "CI_BUILD_TRIGGERED", "value" => "true", "public" => true }, { "key" => "CI_BUILD_TRIGGERED", "value" => "true", "public" => true },
{ "key" => "DB_NAME", "value" => "postgres", "public" => true }, { "key" => "DB_NAME", "value" => "postgres", "public" => true },
{ "key" => "SECRET_KEY", "value" => "secret_value", "public" => false }, { "key" => "SECRET_KEY", "value" => "secret_value", "public" => false },
{ "key" => "TRIGGER_KEY", "value" => "TRIGGER_VALUE", "public" => false }, { "key" => "TRIGGER_KEY", "value" => "TRIGGER_VALUE", "public" => false }
]) )
end end
it "returns dependent builds" do it "returns dependent builds" do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment