Merge branch '21480-parallel-job-keyword-mvc' into 'master'

Resolve "`parallel` job keyword MVC"

Closes #21480

See merge request gitlab-org/gitlab-ce!22631

app/models/ci/build.rb

@@ -815,7 +815,7 @@ module Ci
       end
     end
 
-    def predefined_variables
+    def predefined_variables # rubocop:disable Metrics/AbcSize
       Gitlab::Ci::Variables::Collection.new.tap do |variables|
         variables.append(key: 'CI', value: 'true')
         variables.append(key: 'GITLAB_CI', value: 'true')
@@ -835,6 +835,8 @@ module Ci
         variables.append(key: "CI_COMMIT_TAG", value: ref) if tag?
         variables.append(key: "CI_PIPELINE_TRIGGERED", value: 'true') if trigger_request
         variables.append(key: "CI_JOB_MANUAL", value: 'true') if action?
+        variables.append(key: "CI_NODE_INDEX", value: self.options[:instance].to_s) if self.options&.include?(:instance)
+        variables.append(key: "CI_NODE_TOTAL", value: (self.options&.dig(:parallel) || 1).to_s)
         variables.concat(legacy_variables)
       end
     end

changelogs/unreleased/21480-parallel-job-keyword-mvc.yml

+---
+title: Implement parallel job keyword.
+merge_request: 22631
+author:
+type: added

doc/ci/variables/README.md

@@ -65,6 +65,8 @@ future GitLab releases.**
 | **CI_JOB_NAME**                 | 9.0    | 0.5    | The name of the job as defined in `.gitlab-ci.yml` |
 | **CI_JOB_STAGE**                | 9.0    | 0.5    | The name of the stage as defined in `.gitlab-ci.yml` |
 | **CI_JOB_TOKEN**                | 9.0    | 1.2    | Token used for authenticating with the [GitLab Container Registry][registry] and downloading [dependent repositories][dependent-repositories] |
+| **CI_NODE_INDEX**               | 11.5   | all    | Index of the job in the job set. If the job is not parallelized, this variable is not set. |
+| **CI_NODE_TOTAL**               | 11.5   | all    | Total number of instances of this job running in parallel. If the job is not parallelized, this variable is set to `1`. |
 | **CI_JOB_URL**                  | 11.1   | 0.5    | Job details URL |
 | **CI_REPOSITORY_URL**           | 9.0    | all    | The URL to clone the Git repository |
 | **CI_RUNNER_DESCRIPTION**       | 8.10   | 0.5    | The description of the runner as saved in GitLab |

doc/ci/yaml/README.md

@@ -75,6 +75,7 @@ A job is defined by a list of parameters that define the job behavior.
 | environment   | no       | Defines a name of environment to which deployment is done by this job |
 | coverage      | no       | Define code coverage settings for a given job |
 | retry         | no       | Define how many times a job can be auto-retried in case of a failure |
+| parallel      | no       | Defines how many instances of a job should be run in parallel |
 
 ### `extends`
 
@@ -1451,6 +1452,26 @@ test:
   retry: 2
 ```
 
+## `parallel`
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/22631) in GitLab 11.5.
+
+`parallel` allows you to configure how many instances of a job to run in
+parallel. This value has to be greater than or equal to two (2).
+
+This creates N instances of the same job that run in parallel. They're named
+sequentially from `job_name 1/N` to `job_name N/N`.
+
+For every job, `CI_NODE_INDEX` and `CI_NODE_TOTAL` [environment variables](../variables/README.html#predefined-variables-environment-variables) are set.
+
+A simple example:
+
+```yaml
+test:
+  script: rspec
+  parallel: 5
+```
+
 ## `include`
 
 > Introduced in [GitLab Edition Premium][ee] 10.5.
@@ -2034,4 +2055,4 @@ CI with various languages.
 [schedules]: ../../user/project/pipelines/schedules.md
 [variables-expressions]: ../variables/README.md#variables-expressions
 [ee]: https://about.gitlab.com/gitlab-ee/
-[gitlab-versions]: https://about.gitlab.com/products/
\ No newline at end of file
+[gitlab-versions]: https://about.gitlab.com/products/

lib/gitlab/ci/config/entry/job.rb

@@ -14,7 +14,7 @@ module Gitlab
           ALLOWED_KEYS = %i[tags script only except type image services
                             allow_failure type stage when start_in artifacts cache
                             dependencies before_script after_script variables
-                            environment coverage retry extends].freeze
+                            environment coverage retry parallel extends].freeze
 
           validations do
             validates :config, allowed_keys: ALLOWED_KEYS
@@ -29,6 +29,8 @@ module Gitlab
               validates :retry, numericality: { only_integer: true,
                                                 greater_than_or_equal_to: 0,
                                                 less_than_or_equal_to: 2 }
+              validates :parallel, numericality: { only_integer: true,
+                                                   greater_than_or_equal_to: 2 }
               validates :when,
                 inclusion: { in: %w[on_success on_failure always manual delayed],
                              message: 'should be on_success, on_failure, ' \
@@ -79,17 +81,18 @@ module Gitlab
             description: 'Artifacts configuration for this job.'
 
           entry :environment, Entry::Environment,
-               description: 'Environment configuration for this job.'
+            description: 'Environment configuration for this job.'
 
           entry :coverage, Entry::Coverage,
-               description: 'Coverage configuration for this job.'
+            description: 'Coverage configuration for this job.'
 
           helpers :before_script, :script, :stage, :type, :after_script,
                   :cache, :image, :services, :only, :except, :variables,
-                  :artifacts, :commands, :environment, :coverage, :retry
+                  :artifacts, :commands, :environment, :coverage, :retry,
+                  :parallel
 
           attributes :script, :tags, :allow_failure, :when, :dependencies,
-                     :retry, :extends, :start_in
+                     :retry, :parallel, :extends, :start_in
 
           def compose!(deps = nil)
             super do
@@ -158,6 +161,7 @@ module Gitlab
               environment_name: environment_defined? ? environment_value[:name] : nil,
               coverage: coverage_defined? ? coverage_value : nil,
               retry: retry_defined? ? retry_value.to_i : nil,
+              parallel: parallel_defined? ? parallel_value.to_i : nil,
               artifacts: artifacts_value,
               after_script: after_script_value,
               ignore: ignored? }

lib/gitlab/ci/config/normalizer.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      class Normalizer
+        def initialize(jobs_config)
+          @jobs_config = jobs_config
+        end
+
+        def normalize_jobs
+          extract_parallelized_jobs
+          parallelized_config = parallelize_jobs
+          parallelize_dependencies(parallelized_config)
+        end
+
+        private
+
+        def extract_parallelized_jobs
+          @parallelized_jobs = {}
+
+          @jobs_config.each do |job_name, config|
+            if config[:parallel]
+              @parallelized_jobs[job_name] = self.class.parallelize_job_names(job_name, config[:parallel])
+            end
+          end
+
+          @parallelized_jobs
+        end
+
+        def parallelize_jobs
+          @jobs_config.each_with_object({}) do |(job_name, config), hash|
+            if @parallelized_jobs.key?(job_name)
+              @parallelized_jobs[job_name].each { |name, index| hash[name.to_sym] = config.merge(name: name, instance: index) }
+            else
+              hash[job_name] = config
+            end
+
+            hash
+          end
+        end
+
+        def parallelize_dependencies(parallelized_config)
+          parallelized_config.each_with_object({}) do |(job_name, config), hash|
+            parallelized_job_names = @parallelized_jobs.keys.map(&:to_s)
+            if config[:dependencies] && (intersection = config[:dependencies] & parallelized_job_names).any?
+              deps = intersection.map { |dep| @parallelized_jobs[dep.to_sym].map(&:first) }.flatten
+              hash[job_name] = config.merge(dependencies: deps)
+            else
+              hash[job_name] = config
+            end
+
+            hash
+          end
+        end
+
+        def self.parallelize_job_names(name, total)
+          Array.new(total) { |index| ["#{name} #{index + 1}/#{total}", index + 1] }
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/yaml_processor.rb

@@ -52,6 +52,8 @@ module Gitlab
             after_script: job[:after_script],
             environment: job[:environment],
             retry: job[:retry],
+            parallel: job[:parallel],
+            instance: job[:instance],
             start_in: job[:start_in]
           }.compact }
       end
@@ -104,7 +106,7 @@ module Gitlab
         ##
         # Jobs
         #
-        @jobs = @ci_config.jobs
+        @jobs = Ci::Config::Normalizer.new(@ci_config.jobs).normalize_jobs
 
         @jobs.each do |name, job|
           # logical validation for job

spec/lib/gitlab/ci/config/entry/job_spec.rb

-require 'fast_spec_helper'
-require_dependency 'active_model'
+require 'spec_helper'
 
 describe Gitlab::Ci::Config::Entry::Job do
   let(:entry) { described_class.new(config, name: :rspec) }
@@ -138,6 +137,36 @@ describe Gitlab::Ci::Config::Entry::Job do
         end
       end
 
+      context 'when parallel value is not correct' do
+        context 'when it is not a numeric value' do
+          let(:config) { { parallel: true } }
+
+          it 'returns error about invalid type' do
+            expect(entry).not_to be_valid
+            expect(entry.errors).to include 'job parallel is not a number'
+          end
+        end
+
+        context 'when it is lower than two' do
+          let(:config) { { parallel: 1 } }
+
+          it 'returns error about value too low' do
+            expect(entry).not_to be_valid
+            expect(entry.errors)
+              .to include 'job parallel must be greater than or equal to 2'
+          end
+        end
+
+        context 'when it is not an integer' do
+          let(:config) { { parallel: 1.5 } }
+
+          it 'returns error about wrong value' do
+            expect(entry).not_to be_valid
+            expect(entry.errors).to include 'job parallel must be an integer'
+          end
+        end
+      end
+
       context 'when delayed job' do
         context 'when start_in is specified' do
           let(:config) { { script: 'echo', when: 'delayed', start_in: '1 day' } }

spec/lib/gitlab/ci/config/normalizer_spec.rb

+# frozen_string_literal: true
+
+require 'fast_spec_helper'
+
+describe Gitlab::Ci::Config::Normalizer do
+  let(:job_name) { :rspec }
+  let(:job_config) { { script: 'rspec', parallel: 5, name: 'rspec' } }
+  let(:config) { { job_name => job_config } }
+
+  describe '.normalize_jobs' do
+    subject { described_class.new(config).normalize_jobs }
+
+    it 'does not have original job' do
+      is_expected.not_to include(job_name)
+    end
+
+    it 'has parallelized jobs' do
+      job_names = [:"rspec 1/5", :"rspec 2/5", :"rspec 3/5", :"rspec 4/5", :"rspec 5/5"]
+
+      is_expected.to include(*job_names)
+    end
+
+    it 'sets job instance in options' do
+      expect(subject.values).to all(include(:instance))
+    end
+
+    it 'parallelizes jobs with original config' do
+      original_config = config[job_name].except(:name)
+      configs = subject.values.map { |config| config.except(:name, :instance) }
+
+      expect(configs).to all(eq(original_config))
+    end
+
+    context 'when there is a job with a slash in it' do
+      let(:job_name) { :"rspec 35/2" }
+
+      it 'properly parallelizes job names' do
+        job_names = [:"rspec 35/2 1/5", :"rspec 35/2 2/5", :"rspec 35/2 3/5", :"rspec 35/2 4/5", :"rspec 35/2 5/5"]
+
+        is_expected.to include(*job_names)
+      end
+    end
+
+    context 'when jobs depend on parallelized jobs' do
+      let(:config) { { job_name => job_config, other_job: { script: 'echo 1', dependencies: [job_name.to_s] } } }
+
+      it 'parallelizes dependencies' do
+        job_names = ["rspec 1/5", "rspec 2/5", "rspec 3/5", "rspec 4/5", "rspec 5/5"]
+
+        expect(subject[:other_job][:dependencies]).to include(*job_names)
+      end
+
+      it 'does not include original job name in dependencies' do
+        expect(subject[:other_job][:dependencies]).not_to include(job_name)
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/yaml_processor_spec.rb

@@ -645,6 +645,33 @@ module Gitlab
         end
       end
 
+      describe 'Parallel' do
+        context 'when job is parallelized' do
+          let(:parallel) { 5 }
+
+          let(:config) do
+            YAML.dump(rspec: { script: 'rspec',
+                               parallel: parallel })
+          end
+
+          it 'returns parallelized jobs' do
+            config_processor = Gitlab::Ci::YamlProcessor.new(config)
+            builds = config_processor.stage_builds_attributes('test')
+            build_options = builds.map { |build| build[:options] }
+
+            expect(builds.size).to eq(5)
+            expect(build_options).to all(include(:instance, parallel: parallel))
+          end
+
+          it 'does not have the original job' do
+            config_processor = Gitlab::Ci::YamlProcessor.new(config)
+            builds = config_processor.stage_builds_attributes('test')
+
+            expect(builds).not_to include(:rspec)
+          end
+        end
+      end
+
       describe 'cache' do
         context 'when cache definition has unknown keys' do
           it 'raises relevant validation error' do

spec/models/ci/build_spec.rb

@@ -2015,6 +2015,7 @@ describe Ci::Build do
         { key: 'CI_COMMIT_BEFORE_SHA', value: build.before_sha, public: true },
         { key: 'CI_COMMIT_REF_NAME', value: build.ref, public: true },
         { key: 'CI_COMMIT_REF_SLUG', value: build.ref_slug, public: true },
+        { key: 'CI_NODE_TOTAL', value: '1', public: true },
         { key: 'CI_BUILD_REF', value: build.sha, public: true },
         { key: 'CI_BUILD_BEFORE_SHA', value: build.before_sha, public: true },
         { key: 'CI_BUILD_REF_NAME', value: build.ref, public: true },
@@ -2476,6 +2477,29 @@ describe Ci::Build do
       end
     end
 
+    context 'when build is parallelized' do
+      let(:total) { 5 }
+      let(:index) { 3 }
+
+      before do
+        build.options[:parallel] = total
+        build.options[:instance] = index
+        build.name = "#{build.name} #{index}/#{total}"
+      end
+
+      it 'includes CI_NODE_INDEX' do
+        is_expected.to include(
+          { key: 'CI_NODE_INDEX', value: index.to_s, public: true }
+        )
+      end
+
+      it 'includes correct CI_NODE_TOTAL' do
+        is_expected.to include(
+          { key: 'CI_NODE_TOTAL', value: total.to_s, public: true }
+        )
+      end
+    end
+
     describe 'variables ordering' do
       context 'when variables hierarchy is stubbed' do
         let(:build_pre_var) { { key: 'build', value: 'value', public: true } }