Commit 377a0052 authored by Marius Bobin's avatar Marius Bobin

Add !reference tag to the CI syntax for merging YAML arrays

Related to https://gitlab.com/gitlab-org/gitlab/-/issues/266173
parent 918d107f
---
name: ci_custom_yaml_tags
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/52104
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/300155
milestone: '13.9'
type: development
group: group::pipeline authoring
default_enabled: false
......@@ -90,6 +90,7 @@ def instrument_classes(instrumentation)
instrumentation.instrument_methods(Gitlab::Highlight)
instrumentation.instrument_instance_methods(Gitlab::Highlight)
instrumentation.instrument_instance_method(Gitlab::Ci::Config::Yaml::Tags::Resolver, :to_hash)
Gitlab.ee do
instrumentation.instrument_instance_methods(Elastic::Latest::GitInstanceProxy)
......
......@@ -13,7 +13,8 @@ module Gitlab
RESCUE_ERRORS = [
Gitlab::Config::Loader::FormatError,
Extendable::ExtensionError,
External::Processor::IncludeError
External::Processor::IncludeError,
Config::Yaml::Tags::TagError
].freeze
attr_reader :root
......@@ -89,6 +90,24 @@ module Gitlab
end
def build_config(config)
if ::Feature.enabled?(:ci_custom_yaml_tags, @context.project, default_enabled: :yaml)
build_config_with_custom_tags(config)
else
build_config_without_custom_tags(config)
end
end
def build_config_with_custom_tags(config)
initial_config = Config::Yaml.load!(config, project: @context.project)
initial_config = Config::External::Processor.new(initial_config, @context).perform
initial_config = Config::Extendable.new(initial_config).to_hash
initial_config = Config::Yaml::Tags::Resolver.new(initial_config).to_hash
initial_config = Config::EdgeStagesInjector.new(initial_config).to_hash
initial_config
end
def build_config_without_custom_tags(config)
initial_config = Gitlab::Config::Loader::Yaml.new(config).load!
initial_config = Config::External::Processor.new(initial_config, @context).perform
initial_config = Config::Extendable.new(initial_config).to_hash
......
......@@ -60,7 +60,11 @@ module Gitlab
def content_hash
strong_memoize(:content_yaml) do
Gitlab::Config::Loader::Yaml.new(content).load!
if ::Feature.enabled?(:ci_custom_yaml_tags, context.project, default_enabled: :yaml)
::Gitlab::Ci::Config::Yaml.load!(content)
else
Gitlab::Config::Loader::Yaml.new(content).load!
end
end
rescue Gitlab::Config::Loader::FormatError
nil
......
# frozen_string_literal: true
module Gitlab
module Ci
class Config
module Yaml
AVAILABLE_TAGS = [Config::Yaml::Tags::Reference].freeze
class << self
def load!(content, project: nil)
ensure_custom_tags
Gitlab::Config::Loader::Yaml.new(content, additional_permitted_classes: AVAILABLE_TAGS).load!
end
private
def ensure_custom_tags
@ensure_custom_tags ||= begin
AVAILABLE_TAGS.each { |klass| Psych.add_tag(klass.tag, klass) }
true
end
end
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Ci
class Config
module Yaml
module Tags
TagError = Class.new(StandardError)
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Ci
class Config
module Yaml
module Tags
class Base
CircularReferenceError = Class.new(Tags::TagError)
NotValidError = Class.new(Tags::TagError)
extend ::Gitlab::Utils::Override
attr_accessor :resolved_status, :resolved_value, :data
def self.tag
raise NotImplementedError
end
# Only one of the `seq`, `scalar`, `map` fields is available.
def init_with(coder)
@data = {
tag: coder.tag, # This is the custom YAML tag, like !reference or !flatten
style: coder.style,
seq: coder.seq, # This holds Array data
scalar: coder.scalar, # This holds data of basic types, like String.
map: coder.map # This holds Hash data.
}
end
def valid?
raise NotImplementedError
end
def resolve(resolver)
raise NotValidError, validation_error_message unless valid?
raise CircularReferenceError, circular_error_message if resolving?
return resolved_value if resolved?
self.resolved_status = :in_progress
self.resolved_value = _resolve(resolver)
self.resolved_status = :done
resolved_value
end
private
def _resolve(resolver)
raise NotImplementedError
end
def resolved?
resolved_status == :done
end
def resolving?
resolved_status == :in_progress
end
def circular_error_message
"#{data[:tag]} #{data[:seq].inspect} is part of a circular chain"
end
def validation_error_message
"#{data[:tag]} #{(data[:scalar].presence || data[:map].presence || data[:seq]).inspect} is not valid"
end
end
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Ci
class Config
module Yaml
module Tags
class Reference < Base
MissingReferenceError = Class.new(Tags::TagError)
def self.tag
'!reference'
end
override :valid?
def valid?
data[:seq].is_a?(Array) &&
!data[:seq].empty? &&
data[:seq].all? { |identifier| identifier.is_a?(String) }
end
private
def location
data[:seq].to_a.map(&:to_sym)
end
override :_resolve
def _resolve(resolver)
object = resolver.config.dig(*location)
value = resolver.deep_resolve(object)
raise MissingReferenceError, missing_ref_error_message unless value
value
end
def missing_ref_error_message
"#{data[:tag]} #{data[:seq].inspect} could not be found"
end
end
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Ci
class Config
module Yaml
module Tags
# This class is the entry point for transforming custom YAML tags back
# into primitive objects.
# Usage: `Resolver.new(a_hash_including_custom_tag_objects).to_hash`
#
class Resolver
attr_reader :config
def initialize(config)
@config = config.deep_dup
end
def to_hash
deep_resolve(config)
end
def deep_resolve(object)
case object
when Array
object.map(&method(:resolve_wrapper))
when Hash
object.deep_transform_values(&method(:resolve_wrapper))
else
resolve_wrapper(object)
end
end
def resolve_wrapper(object)
if object.respond_to?(:resolve)
object.resolve(self)
else
object
end
end
end
end
end
end
end
end
......@@ -12,8 +12,12 @@ module Gitlab
MAX_YAML_SIZE = 1.megabyte
MAX_YAML_DEPTH = 100
def initialize(config)
@config = YAML.safe_load(config, [Symbol], [], true)
def initialize(config, additional_permitted_classes: [])
@config = YAML.safe_load(config,
permitted_classes: [Symbol, *additional_permitted_classes],
permitted_symbols: [],
aliases: true
)
rescue Psych::Exception => e
raise Loader::FormatError, e.message
end
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::Ci::Config::Yaml::Tags::Reference do
let(:config) do
Gitlab::Ci::Config::Yaml.load!(yaml)
end
describe '.tag' do
it 'implements the tag method' do
expect(described_class.tag).to eq('!reference')
end
end
describe '#resolve' do
subject { Gitlab::Ci::Config::Yaml::Tags::Resolver.new(config).to_hash }
context 'with circular references' do
let(:yaml) do
<<~YML
a: !reference [b]
b: !reference [a]
YML
end
it 'raises CircularReferenceError' do
expect { subject }.to raise_error Gitlab::Ci::Config::Yaml::Tags::TagError, '!reference ["b"] is part of a circular chain'
end
end
context 'with nested circular references' do
let(:yaml) do
<<~YML
a: !reference [b, c]
b: { c: !reference [d, e, f] }
d: { e: { f: !reference [a] } }
YML
end
it 'raises CircularReferenceError' do
expect { subject }.to raise_error Gitlab::Ci::Config::Yaml::Tags::TagError, '!reference ["b", "c"] is part of a circular chain'
end
end
context 'with missing references' do
let(:yaml) { 'a: !reference [b]' }
it 'raises MissingReferenceError' do
expect { subject }.to raise_error Gitlab::Ci::Config::Yaml::Tags::TagError, '!reference ["b"] could not be found'
end
end
context 'with invalid references' do
using RSpec::Parameterized::TableSyntax
where(:yaml, :error_message) do
'a: !reference' | '!reference [] is not valid'
'a: !reference str' | '!reference "str" is not valid'
'a: !reference 1' | '!reference "1" is not valid'
'a: !reference [1]' | '!reference [1] is not valid'
'a: !reference { b: c }' | '!reference {"b"=>"c"} is not valid'
end
with_them do
it 'raises an error' do
expect { subject }.to raise_error Gitlab::Ci::Config::Yaml::Tags::TagError, error_message
end
end
end
context 'with arrays' do
let(:yaml) do
<<~YML
a: { b: [1, 2] }
c: { d: { e: [3, 4] } }
f: { g: [ !reference [a, b], 5, !reference [c, d, e]] }
YML
end
it { is_expected.to match(a_hash_including({ f: { g: [[1, 2], 5, [3, 4]] } })) }
end
context 'with hashes' do
context 'when referencing an entire hash' do
let(:yaml) do
<<~YML
a: { b: { c: 'c', d: 'd' } }
e: { f: !reference [a, b] }
YML
end
it { is_expected.to match(a_hash_including({ e: { f: { c: 'c', d: 'd' } } })) }
end
context 'when referencing only a hash value' do
let(:yaml) do
<<~YML
a: { b: { c: 'c', d: 'd' } }
e: { f: { g: !reference [a, b, c], h: 'h' } }
i: !reference [e, f]
YML
end
it { is_expected.to match(a_hash_including({ i: { g: 'c', h: 'h' } })) }
end
context 'when referencing a value before its definition' do
let(:yaml) do
<<~YML
a: { b: !reference [c, d] }
g: { h: { i: 'i', j: 1 } }
c: { d: { e: !reference [g, h, j], f: 'f' } }
YML
end
it { is_expected.to match(a_hash_including({ a: { b: { e: 1, f: 'f' } } })) }
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::Ci::Config::Yaml::Tags::Resolver do
let(:config) do
Gitlab::Ci::Config::Yaml.load!(yaml)
end
describe '#to_hash' do
subject { described_class.new(config).to_hash }
context 'when referencing deeply nested arrays' do
let(:yaml_templates) do
<<~YML
.job-1:
script:
- echo doing step 1 of job 1
- echo doing step 2 of job 1
.job-2:
script:
- echo doing step 1 of job 2
- !reference [.job-1, script]
- echo doing step 2 of job 2
.job-3:
script:
- echo doing step 1 of job 3
- !reference [.job-2, script]
- echo doing step 2 of job 3
YML
end
let(:job_yaml) do
<<~YML
test:
script:
- echo preparing to test
- !reference [.job-3, script]
- echo test finished
YML
end
shared_examples 'expands references' do
it 'expands the references' do
is_expected.to match({
'.job-1': {
script: [
'echo doing step 1 of job 1',
'echo doing step 2 of job 1'
]
},
'.job-2': {
script: [
'echo doing step 1 of job 2',
[
'echo doing step 1 of job 1',
'echo doing step 2 of job 1'
],
'echo doing step 2 of job 2'
]
},
'.job-3': {
script: [
'echo doing step 1 of job 3',
[
'echo doing step 1 of job 2',
[
'echo doing step 1 of job 1',
'echo doing step 2 of job 1'
],
'echo doing step 2 of job 2'
],
'echo doing step 2 of job 3'
]
},
test: {
script: [
'echo preparing to test',
[
'echo doing step 1 of job 3',
[
'echo doing step 1 of job 2',
[
'echo doing step 1 of job 1',
'echo doing step 2 of job 1'
],
'echo doing step 2 of job 2'
],
'echo doing step 2 of job 3'
],
'echo test finished'
]
}
})
end
end
context 'when templates are defined before the job' do
let(:yaml) do
<<~YML
#{yaml_templates}
#{job_yaml}
YML
end
it_behaves_like 'expands references'
end
context 'when templates are defined after the job' do
let(:yaml) do
<<~YML
#{job_yaml}
#{yaml_templates}
YML
end
it_behaves_like 'expands references'
end
end
end
end
......@@ -263,6 +263,26 @@ RSpec.describe Gitlab::Ci::Config do
end
end
end
context 'when yaml uses circular !reference' do
let(:yml) do
<<~YAML
job-1:
script:
- !reference [job-2, before_script]
job-2:
before_script: !reference [job-1, script]
YAML
end
it 'raises error' do
expect { config }.to raise_error(
described_class::ConfigError,
/\!reference \["job-2", "before_script"\] is part of a circular chain/
)
end
end
end
context "when using 'include' directive" do
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Ci::CreatePipelineService do
describe '!reference tags' do
let_it_be(:project) { create(:project, :repository) }
let_it_be(:user) { project.owner }
let(:ref) { 'refs/heads/master' }
let(:source) { :push }
let(:service) { described_class.new(project, user, { ref: ref }) }
let(:pipeline) { service.execute(source) }
before do
stub_ci_pipeline_yaml_file(config)
end
context 'with valid config' do
let(:config) do
<<~YAML
.job-1:
script:
- echo doing step 1 of job 1
.job-2:
before_script:
- ls
script: !reference [.job-1, script]
job:
before_script: !reference [.job-2, before_script]
script:
- echo doing my first step
- !reference [.job-2, script]
- echo doing my last step
YAML
end
it 'creates a pipeline' do
expect(pipeline).to be_persisted
expect(pipeline.builds.first.options).to match(a_hash_including({
'before_script' => ['ls'],
'script' => [
'echo doing my first step',
'echo doing step 1 of job 1',
'echo doing my last step'
]
}))
end
end
context 'with invalid config' do
let(:config) do
<<~YAML
job-1:
script:
- echo doing step 1 of job 1
- !reference [job-3, script]
job-2:
script:
- echo doing step 1 of job 2
- !reference [job-3, script]
job-3:
script:
- echo doing step 1 of job 3
- !reference [job-1, script]
YAML
end
it 'creates a pipeline without builds' do
expect(pipeline).to be_persisted
expect(pipeline.builds).to be_empty
expect(pipeline.yaml_errors).to eq("!reference [\"job-3\", \"script\"] is part of a circular chain")
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment