Commit 6f6cd4f4 authored by Grzegorz Bizon's avatar Grzegorz Bizon

Merge branch '21480-parallel-job-keyword-mvc' into 'master'

Resolve "`parallel` job keyword MVC"

Closes #21480

See merge request gitlab-org/gitlab-ce!22631
parents b1fae097 7366c319
......@@ -815,7 +815,7 @@ module Ci
end
end
def predefined_variables
def predefined_variables # rubocop:disable Metrics/AbcSize
Gitlab::Ci::Variables::Collection.new.tap do |variables|
variables.append(key: 'CI', value: 'true')
variables.append(key: 'GITLAB_CI', value: 'true')
......@@ -835,6 +835,8 @@ module Ci
variables.append(key: "CI_COMMIT_TAG", value: ref) if tag?
variables.append(key: "CI_PIPELINE_TRIGGERED", value: 'true') if trigger_request
variables.append(key: "CI_JOB_MANUAL", value: 'true') if action?
variables.append(key: "CI_NODE_INDEX", value: self.options[:instance].to_s) if self.options&.include?(:instance)
variables.append(key: "CI_NODE_TOTAL", value: (self.options&.dig(:parallel) || 1).to_s)
variables.concat(legacy_variables)
end
end
......
---
title: Implement parallel job keyword.
merge_request: 22631
author:
type: added
......@@ -65,6 +65,8 @@ future GitLab releases.**
| **CI_JOB_NAME** | 9.0 | 0.5 | The name of the job as defined in `.gitlab-ci.yml` |
| **CI_JOB_STAGE** | 9.0 | 0.5 | The name of the stage as defined in `.gitlab-ci.yml` |
| **CI_JOB_TOKEN** | 9.0 | 1.2 | Token used for authenticating with the [GitLab Container Registry][registry] and downloading [dependent repositories][dependent-repositories] |
| **CI_NODE_INDEX** | 11.5 | all | Index of the job in the job set. If the job is not parallelized, this variable is not set. |
| **CI_NODE_TOTAL** | 11.5 | all | Total number of instances of this job running in parallel. If the job is not parallelized, this variable is set to `1`. |
| **CI_JOB_URL** | 11.1 | 0.5 | Job details URL |
| **CI_REPOSITORY_URL** | 9.0 | all | The URL to clone the Git repository |
| **CI_RUNNER_DESCRIPTION** | 8.10 | 0.5 | The description of the runner as saved in GitLab |
......
......@@ -75,6 +75,7 @@ A job is defined by a list of parameters that define the job behavior.
| environment | no | Defines a name of environment to which deployment is done by this job |
| coverage | no | Define code coverage settings for a given job |
| retry | no | Define how many times a job can be auto-retried in case of a failure |
| parallel | no | Defines how many instances of a job should be run in parallel |
### `extends`
......@@ -1451,6 +1452,26 @@ test:
retry: 2
```
## `parallel`
> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/22631) in GitLab 11.5.
`parallel` allows you to configure how many instances of a job to run in
parallel. This value has to be greater than or equal to two (2).
This creates N instances of the same job that run in parallel. They're named
sequentially from `job_name 1/N` to `job_name N/N`.
For every job, `CI_NODE_INDEX` and `CI_NODE_TOTAL` [environment variables](../variables/README.html#predefined-variables-environment-variables) are set.
A simple example:
```yaml
test:
script: rspec
parallel: 5
```
## `include`
> Introduced in [GitLab Edition Premium][ee] 10.5.
......
......@@ -14,7 +14,7 @@ module Gitlab
ALLOWED_KEYS = %i[tags script only except type image services
allow_failure type stage when start_in artifacts cache
dependencies before_script after_script variables
environment coverage retry extends].freeze
environment coverage retry parallel extends].freeze
validations do
validates :config, allowed_keys: ALLOWED_KEYS
......@@ -29,6 +29,8 @@ module Gitlab
validates :retry, numericality: { only_integer: true,
greater_than_or_equal_to: 0,
less_than_or_equal_to: 2 }
validates :parallel, numericality: { only_integer: true,
greater_than_or_equal_to: 2 }
validates :when,
inclusion: { in: %w[on_success on_failure always manual delayed],
message: 'should be on_success, on_failure, ' \
......@@ -86,10 +88,11 @@ module Gitlab
helpers :before_script, :script, :stage, :type, :after_script,
:cache, :image, :services, :only, :except, :variables,
:artifacts, :commands, :environment, :coverage, :retry
:artifacts, :commands, :environment, :coverage, :retry,
:parallel
attributes :script, :tags, :allow_failure, :when, :dependencies,
:retry, :extends, :start_in
:retry, :parallel, :extends, :start_in
def compose!(deps = nil)
super do
......@@ -158,6 +161,7 @@ module Gitlab
environment_name: environment_defined? ? environment_value[:name] : nil,
coverage: coverage_defined? ? coverage_value : nil,
retry: retry_defined? ? retry_value.to_i : nil,
parallel: parallel_defined? ? parallel_value.to_i : nil,
artifacts: artifacts_value,
after_script: after_script_value,
ignore: ignored? }
......
# frozen_string_literal: true
module Gitlab
module Ci
class Config
class Normalizer
def initialize(jobs_config)
@jobs_config = jobs_config
end
def normalize_jobs
extract_parallelized_jobs
parallelized_config = parallelize_jobs
parallelize_dependencies(parallelized_config)
end
private
def extract_parallelized_jobs
@parallelized_jobs = {}
@jobs_config.each do |job_name, config|
if config[:parallel]
@parallelized_jobs[job_name] = self.class.parallelize_job_names(job_name, config[:parallel])
end
end
@parallelized_jobs
end
def parallelize_jobs
@jobs_config.each_with_object({}) do |(job_name, config), hash|
if @parallelized_jobs.key?(job_name)
@parallelized_jobs[job_name].each { |name, index| hash[name.to_sym] = config.merge(name: name, instance: index) }
else
hash[job_name] = config
end
hash
end
end
def parallelize_dependencies(parallelized_config)
parallelized_config.each_with_object({}) do |(job_name, config), hash|
parallelized_job_names = @parallelized_jobs.keys.map(&:to_s)
if config[:dependencies] && (intersection = config[:dependencies] & parallelized_job_names).any?
deps = intersection.map { |dep| @parallelized_jobs[dep.to_sym].map(&:first) }.flatten
hash[job_name] = config.merge(dependencies: deps)
else
hash[job_name] = config
end
hash
end
end
def self.parallelize_job_names(name, total)
Array.new(total) { |index| ["#{name} #{index + 1}/#{total}", index + 1] }
end
end
end
end
end
......@@ -52,6 +52,8 @@ module Gitlab
after_script: job[:after_script],
environment: job[:environment],
retry: job[:retry],
parallel: job[:parallel],
instance: job[:instance],
start_in: job[:start_in]
}.compact }
end
......@@ -104,7 +106,7 @@ module Gitlab
##
# Jobs
#
@jobs = @ci_config.jobs
@jobs = Ci::Config::Normalizer.new(@ci_config.jobs).normalize_jobs
@jobs.each do |name, job|
# logical validation for job
......
require 'fast_spec_helper'
require_dependency 'active_model'
require 'spec_helper'
describe Gitlab::Ci::Config::Entry::Job do
let(:entry) { described_class.new(config, name: :rspec) }
......@@ -138,6 +137,36 @@ describe Gitlab::Ci::Config::Entry::Job do
end
end
context 'when parallel value is not correct' do
context 'when it is not a numeric value' do
let(:config) { { parallel: true } }
it 'returns error about invalid type' do
expect(entry).not_to be_valid
expect(entry.errors).to include 'job parallel is not a number'
end
end
context 'when it is lower than two' do
let(:config) { { parallel: 1 } }
it 'returns error about value too low' do
expect(entry).not_to be_valid
expect(entry.errors)
.to include 'job parallel must be greater than or equal to 2'
end
end
context 'when it is not an integer' do
let(:config) { { parallel: 1.5 } }
it 'returns error about wrong value' do
expect(entry).not_to be_valid
expect(entry.errors).to include 'job parallel must be an integer'
end
end
end
context 'when delayed job' do
context 'when start_in is specified' do
let(:config) { { script: 'echo', when: 'delayed', start_in: '1 day' } }
......
# frozen_string_literal: true
require 'fast_spec_helper'
describe Gitlab::Ci::Config::Normalizer do
let(:job_name) { :rspec }
let(:job_config) { { script: 'rspec', parallel: 5, name: 'rspec' } }
let(:config) { { job_name => job_config } }
describe '.normalize_jobs' do
subject { described_class.new(config).normalize_jobs }
it 'does not have original job' do
is_expected.not_to include(job_name)
end
it 'has parallelized jobs' do
job_names = [:"rspec 1/5", :"rspec 2/5", :"rspec 3/5", :"rspec 4/5", :"rspec 5/5"]
is_expected.to include(*job_names)
end
it 'sets job instance in options' do
expect(subject.values).to all(include(:instance))
end
it 'parallelizes jobs with original config' do
original_config = config[job_name].except(:name)
configs = subject.values.map { |config| config.except(:name, :instance) }
expect(configs).to all(eq(original_config))
end
context 'when there is a job with a slash in it' do
let(:job_name) { :"rspec 35/2" }
it 'properly parallelizes job names' do
job_names = [:"rspec 35/2 1/5", :"rspec 35/2 2/5", :"rspec 35/2 3/5", :"rspec 35/2 4/5", :"rspec 35/2 5/5"]
is_expected.to include(*job_names)
end
end
context 'when jobs depend on parallelized jobs' do
let(:config) { { job_name => job_config, other_job: { script: 'echo 1', dependencies: [job_name.to_s] } } }
it 'parallelizes dependencies' do
job_names = ["rspec 1/5", "rspec 2/5", "rspec 3/5", "rspec 4/5", "rspec 5/5"]
expect(subject[:other_job][:dependencies]).to include(*job_names)
end
it 'does not include original job name in dependencies' do
expect(subject[:other_job][:dependencies]).not_to include(job_name)
end
end
end
end
......@@ -645,6 +645,33 @@ module Gitlab
end
end
describe 'Parallel' do
context 'when job is parallelized' do
let(:parallel) { 5 }
let(:config) do
YAML.dump(rspec: { script: 'rspec',
parallel: parallel })
end
it 'returns parallelized jobs' do
config_processor = Gitlab::Ci::YamlProcessor.new(config)
builds = config_processor.stage_builds_attributes('test')
build_options = builds.map { |build| build[:options] }
expect(builds.size).to eq(5)
expect(build_options).to all(include(:instance, parallel: parallel))
end
it 'does not have the original job' do
config_processor = Gitlab::Ci::YamlProcessor.new(config)
builds = config_processor.stage_builds_attributes('test')
expect(builds).not_to include(:rspec)
end
end
end
describe 'cache' do
context 'when cache definition has unknown keys' do
it 'raises relevant validation error' do
......
......@@ -2015,6 +2015,7 @@ describe Ci::Build do
{ key: 'CI_COMMIT_BEFORE_SHA', value: build.before_sha, public: true },
{ key: 'CI_COMMIT_REF_NAME', value: build.ref, public: true },
{ key: 'CI_COMMIT_REF_SLUG', value: build.ref_slug, public: true },
{ key: 'CI_NODE_TOTAL', value: '1', public: true },
{ key: 'CI_BUILD_REF', value: build.sha, public: true },
{ key: 'CI_BUILD_BEFORE_SHA', value: build.before_sha, public: true },
{ key: 'CI_BUILD_REF_NAME', value: build.ref, public: true },
......@@ -2476,6 +2477,29 @@ describe Ci::Build do
end
end
context 'when build is parallelized' do
let(:total) { 5 }
let(:index) { 3 }
before do
build.options[:parallel] = total
build.options[:instance] = index
build.name = "#{build.name} #{index}/#{total}"
end
it 'includes CI_NODE_INDEX' do
is_expected.to include(
{ key: 'CI_NODE_INDEX', value: index.to_s, public: true }
)
end
it 'includes correct CI_NODE_TOTAL' do
is_expected.to include(
{ key: 'CI_NODE_TOTAL', value: total.to_s, public: true }
)
end
end
describe 'variables ordering' do
context 'when variables hierarchy is stubbed' do
let(:build_pre_var) { { key: 'build', value: 'value', public: true } }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment