Commit 20702d80 authored by Rémy Coutable's avatar Rémy Coutable

Merge branch 'ci-predefined-variables' into 'master'

Add predefined CI variables to GitLab

## What does this MR do?
This adds predefined CI variables to GitLab for container registry, pipelines, project name, etc. It also makes sure that all currently documented variables are send from GitLab. This is added to follow up on this proposal: https://gitlab.com/gitlab-org/gitlab-ci-multi-runner/issues/185#note_11844286. To migrate almost all variables out of Runner to GitLab to simplify adding a new of variables without the need for changing the GitLab Runner.

## Why was this MR needed?
Our CI variables miss a lot of crucial information that should be easily accessible. This tries to fill this gap.

## What are the relevant issue numbers?
Fixes https://gitlab.com/gitlab-org/gitlab-ci-multi-runner/issues/185, https://gitlab.com/gitlab-org/gitlab-ce/issues/18164, https://gitlab.com/gitlab-org/gitlab-ce/issues/18075.

## Does this MR meet the acceptance criteria?

- [ ] [CHANGELOG](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CHANGELOG) entry added
- [ ] [Documentation created/updated](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/development/doc_styleguide.md)
- [ ] API support added
- Tests
  - [ ] Added for this feature/bug
  - [ ] All builds are passing
- [ ] Conform by the [style guides](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CONTRIBUTING.md#style-guides)
- [ ] Branch has no merge conflicts with `master` (if you do - rebase it please)
- [ ] [Squashed related commits together](https://git-scm.com/book/en/Git-Tools-Rewriting-History#Squashing-Commits)


See merge request !4826
parent 386e8c08
......@@ -23,6 +23,7 @@ v 8.10.0 (unreleased)
- Escape file extension when parsing search results !5141 (winniehell)
- Apply the trusted_proxies config to the rack request object for use with rack_attack
- Upgrade to Rails 4.2.7. !5236
- Extend exposed environment variables for CI builds
- Allow to pull code with deploy key from public projects
- Add Sidekiq queue duration to transaction metrics.
- Add a new column `artifacts_size` to table `ci_builds` !4964
......
......@@ -145,7 +145,15 @@ module Ci
end
def variables
predefined_variables + yaml_variables + project_variables + trigger_variables
variables = predefined_variables
variables += project.predefined_variables
variables += pipeline.predefined_variables
variables += runner.predefined_variables if runner
variables += project.container_registry_variables
variables += yaml_variables
variables += project.secret_variables
variables += trigger_request.user_variables if trigger_request
variables
end
def merge_request
......@@ -430,28 +438,23 @@ module Ci
self.update(erased_by: user, erased_at: Time.now, artifacts_expire_at: nil)
end
def project_variables
project.variables.map do |variable|
{ key: variable.key, value: variable.value, public: false }
end
end
def trigger_variables
if trigger_request && trigger_request.variables
trigger_request.variables.map do |key, value|
{ key: key, value: value, public: false }
end
else
[]
end
end
def predefined_variables
variables = []
variables << { key: :CI_BUILD_TAG, value: ref, public: true } if tag?
variables << { key: :CI_BUILD_NAME, value: name, public: true }
variables << { key: :CI_BUILD_STAGE, value: stage, public: true }
variables << { key: :CI_BUILD_TRIGGERED, value: 'true', public: true } if trigger_request
variables = [
{ key: 'CI', value: 'true', public: true },
{ key: 'GITLAB_CI', value: 'true', public: true },
{ key: 'CI_BUILD_ID', value: id.to_s, public: true },
{ key: 'CI_BUILD_TOKEN', value: token, public: false },
{ key: 'CI_BUILD_REF', value: sha, public: true },
{ key: 'CI_BUILD_BEFORE_SHA', value: before_sha, public: true },
{ key: 'CI_BUILD_REF_NAME', value: ref, public: true },
{ key: 'CI_BUILD_NAME', value: name, public: true },
{ key: 'CI_BUILD_STAGE', value: stage, public: true },
{ key: 'CI_SERVER_NAME', value: 'GitLab', public: true },
{ key: 'CI_SERVER_VERSION', value: Gitlab::VERSION, public: true },
{ key: 'CI_SERVER_REVISION', value: Gitlab::REVISION, public: true }
]
variables << { key: 'CI_BUILD_TAG', value: ref, public: true } if tag?
variables << { key: 'CI_BUILD_TRIGGERED', value: 'true', public: true } if trigger_request
variables
end
......
......@@ -198,6 +198,12 @@ module Ci
Note.for_commit_id(sha)
end
def predefined_variables
[
{ key: 'CI_PIPELINE_ID', value: id.to_s, public: true }
]
end
private
def build_builds_for_stages(stages, user, status, trigger_request)
......
......@@ -114,6 +114,14 @@ module Ci
tag_list.any?
end
def predefined_variables
[
{ key: 'CI_RUNNER_ID', value: id.to_s, public: true },
{ key: 'CI_RUNNER_DESCRIPTION', value: description, public: true },
{ key: 'CI_RUNNER_TAGS', value: tag_list.to_s, public: true }
]
end
private
def tag_constraints
......
......@@ -7,5 +7,13 @@ module Ci
has_many :builds, class_name: 'Ci::Build'
serialize :variables
def user_variables
return [] unless variables
variables.map do |key, value|
{ key: key, value: value, public: false }
end
end
end
end
......@@ -1180,4 +1180,34 @@ class Project < ActiveRecord::Base
def ensure_dir_exist
gitlab_shell.add_namespace(repository_storage_path, namespace.path)
end
def predefined_variables
[
{ key: 'CI_PROJECT_ID', value: id.to_s, public: true },
{ key: 'CI_PROJECT_NAME', value: path, public: true },
{ key: 'CI_PROJECT_PATH', value: path_with_namespace, public: true },
{ key: 'CI_PROJECT_NAMESPACE', value: namespace.path, public: true },
{ key: 'CI_PROJECT_URL', value: web_url, public: true }
]
end
def container_registry_variables
return [] unless Gitlab.config.registry.enabled
variables = [
{ key: 'CI_REGISTRY', value: Gitlab.config.registry.host_port, public: true }
]
if container_registry_enabled?
variables << { key: 'CI_REGISTRY_IMAGE', value: container_registry_repository_url, public: true }
end
variables
end
def secret_variables
variables.map do |variable|
{ key: variable.key, value: variable.value, public: false }
end
end
end
......@@ -18,25 +18,35 @@ The `API_TOKEN` will take the Secure Variable value: `SECURE`.
### Predefined variables (Environment Variables)
| Variable | Runner | Description |
|-------------------------|-----|--------|
| **CI** | 0.4 | Mark that build is executed in CI environment |
| **GITLAB_CI** | all | Mark that build is executed in GitLab CI environment |
| **CI_SERVER** | all | Mark that build is executed in CI environment |
| **CI_SERVER_NAME** | all | CI server that is used to coordinate builds |
| **CI_SERVER_VERSION** | all | Not yet defined |
| **CI_SERVER_REVISION** | all | Not yet defined |
| **CI_BUILD_REF** | all | The commit revision for which project is built |
| **CI_BUILD_TAG** | 0.5 | The commit tag name. Present only when building tags. |
| **CI_BUILD_NAME** | 0.5 | The name of the build as defined in `.gitlab-ci.yml` |
| **CI_BUILD_STAGE** | 0.5 | The name of the stage as defined in `.gitlab-ci.yml` |
| **CI_BUILD_REF_NAME** | all | The branch or tag name for which project is built |
| **CI_BUILD_ID** | all | The unique id of the current build that GitLab CI uses internally |
| **CI_BUILD_REPO** | all | The URL to clone the Git repository |
| **CI_BUILD_TRIGGERED** | 0.5 | The flag to indicate that build was [triggered] |
| **CI_BUILD_TOKEN** | 1.2 | Token used for authenticating with the GitLab Container Registry |
| **CI_PROJECT_ID** | all | The unique id of the current project that GitLab CI uses internally |
| **CI_PROJECT_DIR** | all | The full path where the repository is cloned and where the build is ran |
| Variable | GitLab | Runner | Description |
|-------------------------|--------|--------|-------------|
| **CI** | all | 0.4 | Mark that build is executed in CI environment |
| **GITLAB_CI** | all | all | Mark that build is executed in GitLab CI environment |
| **CI_SERVER** | all | all | Mark that build is executed in CI environment |
| **CI_SERVER_NAME** | all | all | The name of CI server that is used to coordinate builds |
| **CI_SERVER_VERSION** | all | all | GitLab version that is used to schedule builds |
| **CI_SERVER_REVISION** | all | all | GitLab revision that is used to schedule builds |
| **CI_BUILD_ID** | all | all | The unique id of the current build that GitLab CI uses internally |
| **CI_BUILD_REF** | all | all | The commit revision for which project is built |
| **CI_BUILD_TAG** | all | 0.5 | The commit tag name. Present only when building tags. |
| **CI_BUILD_NAME** | all | 0.5 | The name of the build as defined in `.gitlab-ci.yml` |
| **CI_BUILD_STAGE** | all | 0.5 | The name of the stage as defined in `.gitlab-ci.yml` |
| **CI_BUILD_REF_NAME** | all | all | The branch or tag name for which project is built |
| **CI_BUILD_REPO** | all | all | The URL to clone the Git repository |
| **CI_BUILD_TRIGGERED** | all | 0.5 | The flag to indicate that build was [triggered] |
| **CI_BUILD_TOKEN** | all | 1.2 | Token used for authenticating with the GitLab Container Registry |
| **CI_PIPELINE_ID** | 8.10 | 0.5 | The unique id of the current pipeline that GitLab CI uses internally |
| **CI_PROJECT_ID** | all | all | The unique id of the current project that GitLab CI uses internally |
| **CI_PROJECT_NAME** | 8.10 | 0.5 | The project name that is currently being built |
| **CI_PROJECT_NAMESPACE**| 8.10 | 0.5 | The project namespace that is currently being built |
| **CI_PROJECT_PATH** | 8.10 | 0.5 | The namespace with project name |
| **CI_PROJECT_URL** | 8.10 | 0.5 | The HTTP address to access project |
| **CI_PROJECT_DIR** | all | all | The full path where the repository is cloned and where the build is run |
| **CI_REGISTRY** | 8.10 | 0.5 | If the Container Registry is enabled it returns the address of GitLab's Container Registry |
| **CI_REGISTRY_IMAGE** | 8.10 | 0.5 | If the Container Registry is enabled for the project it returnes the address of the registry tied to the specific project |
| **CI_RUNNER_ID** | 8.10 | 0.5 | The unique id of runner being used |
| **CI_RUNNER_DESCRIPTION** | 8.10 | 0.5 | The description of the runner as saved in GitLab |
| **CI_RUNNER_TAGS** | 8.10 | 0.5 | The defined runner tags |
**Some of the variables are only available when using runner with at least defined version.**
......@@ -46,18 +56,28 @@ Example values:
export CI_BUILD_ID="50"
export CI_BUILD_REF="1ecfd275763eff1d6b4844ea3168962458c9f27a"
export CI_BUILD_REF_NAME="master"
export CI_BUILD_REPO="https://gitlab.com/gitlab-org/gitlab-ce.git"
export CI_BUILD_REPO="https://gitab-ci-token:abcde-1234ABCD5678ef@gitlab.com/gitlab-org/gitlab-ce.git"
export CI_BUILD_TAG="1.0.0"
export CI_BUILD_NAME="spec:other"
export CI_BUILD_STAGE="test"
export CI_BUILD_TRIGGERED="true"
export CI_BUILD_TOKEN="abcde-1234ABCD5678ef"
export CI_PROJECT_DIR="/builds/gitlab-org/gitlab-ce"
export CI_PIPELINE_ID="1000"
export CI_PROJECT_ID="34"
export CI_PROJECT_DIR="/builds/gitlab-org/gitlab-ce"
export CI_PROJECT_NAME="gitlab-ce"
export CI_PROJECT_NAMESPACE="gitlab-org"
export CI_PROJECT_PATH="gitlab-org/gitlab-ce"
export CI_PROJECT_URL="https://gitlab.com/gitlab-org/gitlab-ce"
export CI_REGISTRY="registry.gitlab.com"
export CI_REGISTRY_IMAGE="registry.gitlab.com/gitlab-org/gitlab-ce"
export CI_RUNNER_ID="10"
export CI_RUNNER_DESCRIPTION="my runner"
export CI_RUNNER_TAGS="docker, linux"
export CI_SERVER="yes"
export CI_SERVER_NAME="GitLab CI"
export CI_SERVER_REVISION=""
export CI_SERVER_VERSION=""
export CI_SERVER_NAME="GitLab"
export CI_SERVER_REVISION="8.9.0"
export CI_SERVER_VERSION="70606bf"
```
### YAML-defined variables
......
......@@ -191,79 +191,87 @@ describe Ci::Build, models: true do
end
describe '#variables' do
let(:container_registry_enabled) { false }
let(:predefined_variables) do
[
{ key: :CI_BUILD_NAME, value: 'test', public: true },
{ key: :CI_BUILD_STAGE, value: 'test', public: true },
{ key: 'CI', value: 'true', public: true },
{ key: 'GITLAB_CI', value: 'true', public: true },
{ key: 'CI_BUILD_ID', value: build.id.to_s, public: true },
{ key: 'CI_BUILD_TOKEN', value: build.token, public: false },
{ key: 'CI_BUILD_REF', value: build.sha, public: true },
{ key: 'CI_BUILD_BEFORE_SHA', value: build.before_sha, public: true },
{ key: 'CI_BUILD_REF_NAME', value: 'master', public: true },
{ key: 'CI_BUILD_NAME', value: 'test', public: true },
{ key: 'CI_BUILD_STAGE', value: 'test', public: true },
{ key: 'CI_SERVER_NAME', value: 'GitLab', public: true },
{ key: 'CI_SERVER_VERSION', value: Gitlab::VERSION, public: true },
{ key: 'CI_SERVER_REVISION', value: Gitlab::REVISION, public: true },
{ key: 'CI_PROJECT_ID', value: project.id.to_s, public: true },
{ key: 'CI_PROJECT_NAME', value: project.path, public: true },
{ key: 'CI_PROJECT_PATH', value: project.path_with_namespace, public: true },
{ key: 'CI_PROJECT_NAMESPACE', value: project.namespace.path, public: true },
{ key: 'CI_PROJECT_URL', value: project.web_url, public: true },
{ key: 'CI_PIPELINE_ID', value: pipeline.id.to_s, public: true }
]
end
before do
stub_container_registry_config(enabled: container_registry_enabled, host_port: 'registry.example.com')
end
subject { build.variables }
context 'returns variables' do
let(:yaml_variables) do
[
{ key: :DB_NAME, value: 'postgres', public: true }
]
end
before do
build.yaml_variables = yaml_variables
build.yaml_variables = []
end
it { is_expected.to eq(predefined_variables + yaml_variables) }
context 'for tag' do
let(:tag_variable) do
[
{ key: :CI_BUILD_TAG, value: 'master', public: true }
]
end
it { is_expected.to eq(predefined_variables) }
end
before do
build.update_attributes(tag: true)
end
context 'when build is for tag' do
let(:tag_variable) do
{ key: 'CI_BUILD_TAG', value: 'master', public: true }
end
it { is_expected.to eq(tag_variable + predefined_variables + yaml_variables) }
before do
build.update_attributes(tag: true)
end
context 'and secure variables' do
let(:secure_variables) do
[
{ key: 'SECRET_KEY', value: 'secret_value', public: false }
]
end
it { is_expected.to include(tag_variable) }
end
before do
build.project.variables << Ci::Variable.new(key: 'SECRET_KEY', value: 'secret_value')
end
context 'when secure variable is defined' do
let(:secure_variable) do
{ key: 'SECRET_KEY', value: 'secret_value', public: false }
end
it { is_expected.to eq(predefined_variables + yaml_variables + secure_variables) }
before do
build.project.variables << Ci::Variable.new(key: 'SECRET_KEY', value: 'secret_value')
end
context 'and trigger variables' do
let(:trigger) { create(:ci_trigger, project: project) }
let(:trigger_request) { create(:ci_trigger_request_with_variables, pipeline: pipeline, trigger: trigger) }
let(:trigger_variables) do
[
{ key: :TRIGGER_KEY, value: 'TRIGGER_VALUE', public: false }
]
end
let(:predefined_trigger_variable) do
[
{ key: :CI_BUILD_TRIGGERED, value: 'true', public: true }
]
end
it { is_expected.to include(secure_variable) }
end
before do
build.trigger_request = trigger_request
end
context 'when build is for triggers' do
let(:trigger) { create(:ci_trigger, project: project) }
let(:trigger_request) { create(:ci_trigger_request_with_variables, pipeline: pipeline, trigger: trigger) }
let(:user_trigger_variable) do
{ key: :TRIGGER_KEY, value: 'TRIGGER_VALUE', public: false }
end
let(:predefined_trigger_variable) do
{ key: 'CI_BUILD_TRIGGERED', value: 'true', public: true }
end
it { is_expected.to eq(predefined_variables + predefined_trigger_variable + yaml_variables + secure_variables + trigger_variables) }
end
before do
build.trigger_request = trigger_request
end
it { is_expected.to include(user_trigger_variable) }
it { is_expected.to include(predefined_trigger_variable) }
end
context 'when yaml_variables is undefined' do
context 'when yaml_variables are undefined' do
before do
build.yaml_variables = nil
end
......@@ -282,10 +290,10 @@ describe Ci::Build, models: true do
context 'if config does not have a questioned job' do
let(:config) do
YAML.dump({
test_other: {
script: 'Hello World'
}
})
test_other: {
script: 'Hello World'
}
})
end
it { is_expected.to eq(predefined_variables) }
......@@ -294,13 +302,13 @@ describe Ci::Build, models: true do
context 'if config has variables' do
let(:config) do
YAML.dump({
test: {
script: 'Hello World',
variables: {
KEY: 'value'
}
}
})
test: {
script: 'Hello World',
variables: {
KEY: 'value'
}
}
})
end
let(:variables) do
[{ key: :KEY, value: 'value', public: true }]
......@@ -310,6 +318,58 @@ describe Ci::Build, models: true do
end
end
end
context 'when container registry is enabled' do
let(:container_registry_enabled) { true }
let(:ci_registry) do
{ key: 'CI_REGISTRY', value: 'registry.example.com', public: true }
end
let(:ci_registry_image) do
{ key: 'CI_REGISTRY_IMAGE', value: project.container_registry_repository_url, public: true }
end
context 'and is disabled for project' do
before do
project.update(container_registry_enabled: false)
end
it { is_expected.to include(ci_registry) }
it { is_expected.not_to include(ci_registry_image) }
end
context 'and is enabled for project' do
before do
project.update(container_registry_enabled: true)
end
it { is_expected.to include(ci_registry) }
it { is_expected.to include(ci_registry_image) }
end
end
context 'when runner is assigned to build' do
let(:runner) { create(:ci_runner, description: 'description', tag_list: ['docker', 'linux']) }
before do
build.update(runner: runner)
end
it { is_expected.to include({ key: 'CI_RUNNER_ID', value: runner.id.to_s, public: true }) }
it { is_expected.to include({ key: 'CI_RUNNER_DESCRIPTION', value: 'description', public: true }) }
it { is_expected.to include({ key: 'CI_RUNNER_TAGS', value: 'docker, linux', public: true }) }
end
context 'returns variables in valid order' do
before do
allow(build).to receive(:predefined_variables) { ['predefined'] }
allow(project).to receive(:predefined_variables) { ['project'] }
allow(pipeline).to receive(:predefined_variables) { ['pipeline'] }
allow(build).to receive(:yaml_variables) { ['yaml'] }
allow(project).to receive(:secret_variables) { ['secret'] }
end
it { is_expected.to eq(%w[predefined project pipeline yaml secret]) }
end
end
describe '#has_tags?' do
......
......@@ -73,12 +73,12 @@ describe Ci::API::API do
post ci_api("/builds/register"), token: runner.token, info: { platform: :darwin }
expect(response).to have_http_status(201)
expect(json_response["variables"]).to eq([
expect(json_response["variables"]).to include(
{ "key" => "CI_BUILD_NAME", "value" => "spinach", "public" => true },
{ "key" => "CI_BUILD_STAGE", "value" => "test", "public" => true },
{ "key" => "DB_NAME", "value" => "postgres", "public" => true },
{ "key" => "SECRET_KEY", "value" => "secret_value", "public" => false }
])
)
end
it "returns variables for triggers" do
......@@ -92,14 +92,14 @@ describe Ci::API::API do
post ci_api("/builds/register"), token: runner.token, info: { platform: :darwin }
expect(response).to have_http_status(201)
expect(json_response["variables"]).to eq([
expect(json_response["variables"]).to include(
{ "key" => "CI_BUILD_NAME", "value" => "spinach", "public" => true },
{ "key" => "CI_BUILD_STAGE", "value" => "test", "public" => true },
{ "key" => "CI_BUILD_TRIGGERED", "value" => "true", "public" => true },
{ "key" => "DB_NAME", "value" => "postgres", "public" => true },
{ "key" => "SECRET_KEY", "value" => "secret_value", "public" => false },
{ "key" => "TRIGGER_KEY", "value" => "TRIGGER_VALUE", "public" => false },
])
{ "key" => "TRIGGER_KEY", "value" => "TRIGGER_VALUE", "public" => false }
)
end
it "returns dependent builds" do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment