Add !reference tag to the CI syntax for merging YAML arrays

Related to https://gitlab.com/gitlab-org/gitlab/-/issues/266173

config/feature_flags/development/ci_custom_yaml_tags.yml

+---
+name: ci_custom_yaml_tags
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/52104
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/300155
+milestone: '13.9'
+type: development
+group: group::pipeline authoring
+default_enabled: false

config/initializers/zz_metrics.rb

@@ -90,6 +90,7 @@ def instrument_classes(instrumentation)
 
   instrumentation.instrument_methods(Gitlab::Highlight)
   instrumentation.instrument_instance_methods(Gitlab::Highlight)
+  instrumentation.instrument_instance_method(Gitlab::Ci::Config::Yaml::Tags::Resolver, :to_hash)
 
   Gitlab.ee do
     instrumentation.instrument_instance_methods(Elastic::Latest::GitInstanceProxy)

lib/gitlab/ci/config.rb

@@ -13,7 +13,8 @@ module Gitlab
       RESCUE_ERRORS = [
         Gitlab::Config::Loader::FormatError,
         Extendable::ExtensionError,
-        External::Processor::IncludeError
+        External::Processor::IncludeError,
+        Config::Yaml::Tags::TagError
       ].freeze
 
       attr_reader :root
@@ -89,6 +90,24 @@ module Gitlab
       end
 
       def build_config(config)
+        if ::Feature.enabled?(:ci_custom_yaml_tags, @context.project, default_enabled: :yaml)
+          build_config_with_custom_tags(config)
+        else
+          build_config_without_custom_tags(config)
+        end
+      end
+
+      def build_config_with_custom_tags(config)
+        initial_config = Config::Yaml.load!(config, project: @context.project)
+        initial_config = Config::External::Processor.new(initial_config, @context).perform
+        initial_config = Config::Extendable.new(initial_config).to_hash
+        initial_config = Config::Yaml::Tags::Resolver.new(initial_config).to_hash
+        initial_config = Config::EdgeStagesInjector.new(initial_config).to_hash
+
+        initial_config
+      end
+
+      def build_config_without_custom_tags(config)
         initial_config = Gitlab::Config::Loader::Yaml.new(config).load!
         initial_config = Config::External::Processor.new(initial_config, @context).perform
         initial_config = Config::Extendable.new(initial_config).to_hash

lib/gitlab/ci/config/external/file/base.rb

@@ -60,8 +60,12 @@ module Gitlab
 
             def content_hash
               strong_memoize(:content_yaml) do
+                if ::Feature.enabled?(:ci_custom_yaml_tags, context.project, default_enabled: :yaml)
+                  ::Gitlab::Ci::Config::Yaml.load!(content)
+                else
                   Gitlab::Config::Loader::Yaml.new(content).load!
                 end
+              end
             rescue Gitlab::Config::Loader::FormatError
               nil
             end

lib/gitlab/ci/config/yaml.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Yaml
+        AVAILABLE_TAGS = [Config::Yaml::Tags::Reference].freeze
+
+        class << self
+          def load!(content, project: nil)
+            ensure_custom_tags
+
+            Gitlab::Config::Loader::Yaml.new(content, additional_permitted_classes: AVAILABLE_TAGS).load!
+          end
+
+          private
+
+          def ensure_custom_tags
+            @ensure_custom_tags ||= begin
+              AVAILABLE_TAGS.each { |klass| Psych.add_tag(klass.tag, klass) }
+
+              true
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/config/yaml/tags.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Yaml
+        module Tags
+          TagError = Class.new(StandardError)
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/config/yaml/tags/base.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Yaml
+        module Tags
+          class Base
+            CircularReferenceError = Class.new(Tags::TagError)
+            NotValidError = Class.new(Tags::TagError)
+
+            extend ::Gitlab::Utils::Override
+
+            attr_accessor :resolved_status, :resolved_value, :data
+
+            def self.tag
+              raise NotImplementedError
+            end
+
+            # Only one of the `seq`, `scalar`, `map` fields is available.
+            def init_with(coder)
+              @data = {
+                tag: coder.tag,       # This is the custom YAML tag, like !reference or !flatten
+                style: coder.style,
+                seq: coder.seq,       # This holds Array data
+                scalar: coder.scalar, # This holds data of basic types, like String.
+                map: coder.map        # This holds Hash data.
+              }
+            end
+
+            def valid?
+              raise NotImplementedError
+            end
+
+            def resolve(resolver)
+              raise NotValidError, validation_error_message unless valid?
+              raise CircularReferenceError, circular_error_message if resolving?
+              return resolved_value if resolved?
+
+              self.resolved_status = :in_progress
+              self.resolved_value = _resolve(resolver)
+              self.resolved_status = :done
+              resolved_value
+            end
+
+            private
+
+            def _resolve(resolver)
+              raise NotImplementedError
+            end
+
+            def resolved?
+              resolved_status == :done
+            end
+
+            def resolving?
+              resolved_status == :in_progress
+            end
+
+            def circular_error_message
+              "#{data[:tag]} #{data[:seq].inspect} is part of a circular chain"
+            end
+
+            def validation_error_message
+              "#{data[:tag]} #{(data[:scalar].presence || data[:map].presence || data[:seq]).inspect} is not valid"
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/config/yaml/tags/reference.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Yaml
+        module Tags
+          class Reference < Base
+            MissingReferenceError = Class.new(Tags::TagError)
+
+            def self.tag
+              '!reference'
+            end
+
+            override :valid?
+            def valid?
+              data[:seq].is_a?(Array) &&
+                !data[:seq].empty? &&
+                data[:seq].all? { |identifier| identifier.is_a?(String) }
+            end
+
+            private
+
+            def location
+              data[:seq].to_a.map(&:to_sym)
+            end
+
+            override :_resolve
+            def _resolve(resolver)
+              object = resolver.config.dig(*location)
+              value = resolver.deep_resolve(object)
+
+              raise MissingReferenceError, missing_ref_error_message unless value
+
+              value
+            end
+
+            def missing_ref_error_message
+              "#{data[:tag]} #{data[:seq].inspect} could not be found"
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/config/yaml/tags/resolver.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Yaml
+        module Tags
+          # This class is the entry point for transforming custom YAML tags back
+          # into primitive objects.
+          # Usage: `Resolver.new(a_hash_including_custom_tag_objects).to_hash`
+          #
+          class Resolver
+            attr_reader :config
+
+            def initialize(config)
+              @config = config.deep_dup
+            end
+
+            def to_hash
+              deep_resolve(config)
+            end
+
+            def deep_resolve(object)
+              case object
+              when Array
+                object.map(&method(:resolve_wrapper))
+              when Hash
+                object.deep_transform_values(&method(:resolve_wrapper))
+              else
+                resolve_wrapper(object)
+              end
+            end
+
+            def resolve_wrapper(object)
+              if object.respond_to?(:resolve)
+                object.resolve(self)
+              else
+                object
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/config/loader/yaml.rb

@@ -12,8 +12,12 @@ module Gitlab
         MAX_YAML_SIZE = 1.megabyte
         MAX_YAML_DEPTH = 100
 
-        def initialize(config)
-          @config = YAML.safe_load(config, [Symbol], [], true)
+        def initialize(config, additional_permitted_classes: [])
+          @config = YAML.safe_load(config,
+            permitted_classes: [Symbol, *additional_permitted_classes],
+            permitted_symbols: [],
+            aliases: true
+          )
         rescue Psych::Exception => e
           raise Loader::FormatError, e.message
         end

spec/lib/gitlab/ci/config/yaml/tags/reference_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Ci::Config::Yaml::Tags::Reference do
+  let(:config) do
+    Gitlab::Ci::Config::Yaml.load!(yaml)
+  end
+
+  describe '.tag' do
+    it 'implements the tag method' do
+      expect(described_class.tag).to eq('!reference')
+    end
+  end
+
+  describe '#resolve' do
+    subject { Gitlab::Ci::Config::Yaml::Tags::Resolver.new(config).to_hash }
+
+    context 'with circular references' do
+      let(:yaml) do
+        <<~YML
+        a: !reference [b]
+        b: !reference [a]
+        YML
+      end
+
+      it 'raises CircularReferenceError' do
+        expect { subject }.to raise_error Gitlab::Ci::Config::Yaml::Tags::TagError, '!reference ["b"] is part of a circular chain'
+      end
+    end
+
+    context 'with nested circular references' do
+      let(:yaml) do
+        <<~YML
+        a: !reference [b, c]
+        b: { c: !reference [d, e, f] }
+        d: { e: { f: !reference [a] } }
+        YML
+      end
+
+      it 'raises CircularReferenceError' do
+        expect { subject }.to raise_error Gitlab::Ci::Config::Yaml::Tags::TagError, '!reference ["b", "c"] is part of a circular chain'
+      end
+    end
+
+    context 'with missing references' do
+      let(:yaml) { 'a: !reference [b]' }
+
+      it 'raises MissingReferenceError' do
+        expect { subject }.to raise_error Gitlab::Ci::Config::Yaml::Tags::TagError, '!reference ["b"] could not be found'
+      end
+    end
+
+    context 'with invalid references' do
+      using RSpec::Parameterized::TableSyntax
+
+      where(:yaml, :error_message) do
+        'a: !reference'          | '!reference [] is not valid'
+        'a: !reference str'      | '!reference "str" is not valid'
+        'a: !reference 1'        | '!reference "1" is not valid'
+        'a: !reference [1]'      | '!reference [1] is not valid'
+        'a: !reference { b: c }' | '!reference {"b"=>"c"} is not valid'
+      end
+
+      with_them do
+        it 'raises an error' do
+          expect { subject }.to raise_error Gitlab::Ci::Config::Yaml::Tags::TagError, error_message
+        end
+      end
+    end
+
+    context 'with arrays' do
+      let(:yaml) do
+        <<~YML
+        a: { b: [1, 2] }
+        c: { d: { e: [3, 4] } }
+        f: { g: [ !reference [a, b], 5, !reference [c, d, e]] }
+        YML
+      end
+
+      it { is_expected.to match(a_hash_including({ f: { g: [[1, 2], 5, [3, 4]] } })) }
+    end
+
+    context 'with hashes' do
+      context 'when referencing an entire hash' do
+        let(:yaml) do
+          <<~YML
+          a: { b: { c: 'c', d: 'd' } }
+          e: { f: !reference [a, b] }
+          YML
+        end
+
+        it { is_expected.to match(a_hash_including({ e: { f: { c: 'c', d: 'd' } } })) }
+      end
+
+      context 'when referencing only a hash value' do
+        let(:yaml) do
+          <<~YML
+          a: { b: { c: 'c', d: 'd' } }
+          e: {  f: { g: !reference [a, b, c], h: 'h' } }
+          i: !reference [e, f]
+          YML
+        end
+
+        it { is_expected.to match(a_hash_including({ i: { g: 'c', h: 'h' } })) }
+      end
+
+      context 'when referencing a value before its definition' do
+        let(:yaml) do
+          <<~YML
+          a: { b: !reference [c, d] }
+          g: { h: { i: 'i', j: 1 } }
+          c: { d: { e: !reference [g, h, j], f: 'f' } }
+          YML
+        end
+
+        it { is_expected.to match(a_hash_including({ a: { b: { e: 1, f: 'f' } } })) }
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/config/yaml/tags/resolver_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Ci::Config::Yaml::Tags::Resolver do
+  let(:config) do
+    Gitlab::Ci::Config::Yaml.load!(yaml)
+  end
+
+  describe '#to_hash' do
+    subject { described_class.new(config).to_hash }
+
+    context 'when referencing deeply nested arrays' do
+      let(:yaml_templates) do
+        <<~YML
+        .job-1:
+          script:
+            - echo doing step 1 of job 1
+            - echo doing step 2 of job 1
+
+        .job-2:
+          script:
+            - echo doing step 1 of job 2
+            - !reference [.job-1, script]
+            - echo doing step 2 of job 2
+
+        .job-3:
+          script:
+            - echo doing step 1 of job 3
+            - !reference [.job-2, script]
+            - echo doing step 2 of job 3
+        YML
+      end
+
+      let(:job_yaml) do
+        <<~YML
+        test:
+          script:
+            - echo preparing to test
+            - !reference [.job-3, script]
+            - echo test finished
+        YML
+      end
+
+      shared_examples 'expands references' do
+        it 'expands the references' do
+          is_expected.to match({
+            '.job-1': {
+              script: [
+                'echo doing step 1 of job 1',
+                'echo doing step 2 of job 1'
+              ]
+            },
+            '.job-2': {
+              script: [
+                'echo doing step 1 of job 2',
+                [
+                  'echo doing step 1 of job 1',
+                  'echo doing step 2 of job 1'
+                ],
+                'echo doing step 2 of job 2'
+              ]
+            },
+            '.job-3': {
+              script: [
+                'echo doing step 1 of job 3',
+                [
+                  'echo doing step 1 of job 2',
+                  [
+                    'echo doing step 1 of job 1',
+                    'echo doing step 2 of job 1'
+                  ],
+                  'echo doing step 2 of job 2'
+                ],
+                'echo doing step 2 of job 3'
+              ]
+            },
+            test: {
+              script: [
+                'echo preparing to test',
+                [
+                  'echo doing step 1 of job 3',
+                  [
+                    'echo doing step 1 of job 2',
+                    [
+                      'echo doing step 1 of job 1',
+                      'echo doing step 2 of job 1'
+                    ],
+                    'echo doing step 2 of job 2'
+                  ],
+                  'echo doing step 2 of job 3'
+                ],
+                'echo test finished'
+              ]
+            }
+          })
+        end
+      end
+
+      context 'when templates are defined before the job' do
+        let(:yaml) do
+          <<~YML
+          #{yaml_templates}
+          #{job_yaml}
+          YML
+        end
+
+        it_behaves_like 'expands references'
+      end
+
+      context 'when templates are defined after the job' do
+        let(:yaml) do
+          <<~YML
+          #{job_yaml}
+          #{yaml_templates}
+          YML
+        end
+
+        it_behaves_like 'expands references'
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/config_spec.rb

@@ -263,6 +263,26 @@ RSpec.describe Gitlab::Ci::Config do
         end
       end
     end
+
+    context 'when yaml uses circular !reference' do
+      let(:yml) do
+        <<~YAML
+        job-1:
+          script:
+            - !reference [job-2, before_script]
+
+        job-2:
+          before_script: !reference [job-1, script]
+        YAML
+      end
+
+      it 'raises error' do
+        expect { config }.to raise_error(
+          described_class::ConfigError,
+          /\!reference \["job-2", "before_script"\] is part of a circular chain/
+        )
+      end
+    end
   end
 
   context "when using 'include' directive" do

spec/services/ci/create_pipeline_service/custom_yaml_tags_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+RSpec.describe Ci::CreatePipelineService do
+  describe '!reference tags' do
+    let_it_be(:project) { create(:project, :repository) }
+    let_it_be(:user)    { project.owner }
+
+    let(:ref) { 'refs/heads/master' }
+    let(:source) { :push }
+    let(:service) { described_class.new(project, user, { ref: ref }) }
+    let(:pipeline) { service.execute(source) }
+
+    before do
+      stub_ci_pipeline_yaml_file(config)
+    end
+
+    context 'with valid config' do
+      let(:config) do
+        <<~YAML
+        .job-1:
+          script:
+            - echo doing step 1 of job 1
+
+        .job-2:
+          before_script:
+            - ls
+          script: !reference [.job-1, script]
+
+        job:
+          before_script: !reference [.job-2, before_script]
+          script:
+            - echo doing my first step
+            - !reference [.job-2, script]
+            - echo doing my last step
+        YAML
+      end
+
+      it 'creates a pipeline' do
+        expect(pipeline).to be_persisted
+        expect(pipeline.builds.first.options).to match(a_hash_including({
+          'before_script' => ['ls'],
+          'script' => [
+            'echo doing my first step',
+            'echo doing step 1 of job 1',
+            'echo doing my last step'
+          ]
+        }))
+      end
+    end
+
+    context 'with invalid config' do
+      let(:config) do
+        <<~YAML
+        job-1:
+          script:
+            - echo doing step 1 of job 1
+            - !reference [job-3, script]
+
+        job-2:
+          script:
+            - echo doing step 1 of job 2
+            - !reference [job-3, script]
+
+        job-3:
+          script:
+            - echo doing step 1 of job 3
+            - !reference [job-1, script]
+        YAML
+      end
+
+      it 'creates a pipeline without builds' do
+        expect(pipeline).to be_persisted
+        expect(pipeline.builds).to be_empty
+        expect(pipeline.yaml_errors).to eq("!reference [\"job-3\", \"script\"] is part of a circular chain")
+      end
+    end
+  end
+end