Merge branch 'ci-secrets-syntax' into 'master'

Implement CI syntax for secrets

See merge request gitlab-org/gitlab!33014

ee/lib/ee/gitlab/ci/config/entry/job.rb

+# frozen_string_literal: true
+
+module EE
+  module Gitlab
+    module Ci
+      module Config
+        module Entry
+          module Job
+            extend ActiveSupport::Concern
+            extend ::Gitlab::Utils::Override
+
+            prepended do
+              attributes :secrets
+
+              validations do
+                validates :secrets, absence: { message: 'feature is disabled' }, unless: :secrets_enabled?
+              end
+
+              entry :secrets, ::Gitlab::Ci::Config::Entry::Secrets,
+                description: 'Configured secrets for this job',
+                inherit: false
+            end
+
+            override :value
+            def value
+              super.merge({ secrets: secrets_value }.compact)
+            end
+
+            def secrets_enabled?
+              ::Gitlab::Ci::Features.secrets_syntax_enabled?
+            end
+          end
+        end
+      end
+    end
+  end
+end

ee/lib/ee/gitlab/ci/features.rb

+# frozen_string_literal: true
+
+module EE
+  module Gitlab
+    module Ci
+      module Features
+        extend ActiveSupport::Concern
+
+        prepended do
+          def self.secrets_syntax_enabled?
+            ::Feature.enabled?(:ci_secrets_syntax)
+          end
+        end
+      end
+    end
+  end
+end

ee/lib/gitlab/ci/config/entry/secret.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        ##
+        # Entry that represents a secret definition.
+        #
+        class Secret < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Configurable
+          include ::Gitlab::Config::Entry::Attributable
+
+          ALLOWED_KEYS = %i[vault].freeze
+
+          attributes ALLOWED_KEYS
+
+          entry :vault, Entry::Vault::Secret, description: 'Vault secrets engine configuration'
+
+          validations do
+            validates :config, allowed_keys: ALLOWED_KEYS, required_keys: ALLOWED_KEYS
+          end
+        end
+      end
+    end
+  end
+end

ee/lib/gitlab/ci/config/entry/secrets.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        ##
+        # Entry that represents a secrets definition.
+        #
+        class Secrets < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Validatable
+
+          validations do
+            validates :config, type: Hash
+          end
+
+          def compose!(deps = nil)
+            super do
+              @config.each do |name, config|
+                factory = ::Gitlab::Config::Entry::Factory.new(Entry::Secret)
+                  .value(config || {})
+                  .with(key: name, parent: self, description: "#{name} secret definition") # rubocop:disable CodeReuse/ActiveRecord
+                  .metadata(name: name)
+
+                @entries[name] = factory.create!
+              end
+
+              @entries.each_value do |entry|
+                entry.compose!(deps)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

ee/lib/gitlab/ci/config/entry/vault/engine.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        module Vault
+          ##
+          # Entry that represents Vault secret engine.
+          #
+          class Engine < ::Gitlab::Config::Entry::Node
+            include ::Gitlab::Config::Entry::Validatable
+            include ::Gitlab::Config::Entry::Attributable
+
+            ALLOWED_KEYS = %i[name path].freeze
+
+            attributes ALLOWED_KEYS
+
+            validations do
+              validates :config, type: Hash, allowed_keys: ALLOWED_KEYS, required_keys: ALLOWED_KEYS
+              validates :name, presence: true, type: String
+              validates :path, presence: true, type: String
+            end
+          end
+        end
+      end
+    end
+  end
+end

ee/lib/gitlab/ci/config/entry/vault/secret.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        module Vault
+          ##
+          # Entry that represents Vault secret.
+          #
+          class Secret < ::Gitlab::Config::Entry::Simplifiable
+            strategy :StringStrategy, if: -> (config) { config.is_a?(String) }
+            strategy :HashStrategy, if: -> (config) { config.is_a?(Hash) }
+
+            class UnknownStrategy < ::Gitlab::Config::Entry::Node
+              def errors
+                ["#{location} should be a hash or a string"]
+              end
+            end
+
+            class StringStrategy < ::Gitlab::Config::Entry::Node
+              include ::Gitlab::Config::Entry::Validatable
+
+              validations do
+                validates :config, presence: true
+                validates :config, type: String
+              end
+
+              def value
+                {
+                  engine: {
+                    name: 'kv-v2', path: secret[:engine_path]
+                  },
+                  path: secret[:secret_path],
+                  field: secret[:secret_field]
+                }
+              end
+
+              private
+
+              def secret
+                @secret ||= begin
+                              secret, engine_path = secret_and_engine
+                              secret_path, _, secret_field = secret.rpartition('/')
+
+                              {
+                                engine_path: engine_path,
+                                secret_path: secret_path,
+                                secret_field: secret_field
+                              }
+                            end
+              end
+
+              def secret_and_engine
+                secret, _, engine = config.rpartition('@')
+
+                if secret == ""
+                  secret = config
+                  engine = 'kv-v2'
+                end
+
+                [secret, engine]
+              end
+            end
+
+            class HashStrategy < ::Gitlab::Config::Entry::Node
+              include ::Gitlab::Config::Entry::Configurable
+              include ::Gitlab::Config::Entry::Attributable
+
+              ALLOWED_KEYS = %i[engine path field].freeze
+
+              attributes ALLOWED_KEYS
+
+              entry :engine, Entry::Vault::Engine, description: 'Vault secrets engine configuration'
+
+              validations do
+                validates :config, allowed_keys: ALLOWED_KEYS
+                validates :path, presence: true, type: String
+                validates :field, presence: true, type: String
+                validates :engine, presence: true, type: Hash
+              end
+
+              def value
+                {
+                  engine: engine_value,
+                  path: path,
+                  field: field
+                }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

ee/spec/lib/gitlab/ci/config/entry/job_spec.rb

@@ -6,11 +6,40 @@ RSpec.describe Gitlab::Ci::Config::Entry::Job do
   let(:entry) { described_class.new(config, name: :rspec) }
 
   describe 'validations' do
-    before do
-      entry.compose!
+    context 'when entry value is correct' do
+      context 'when has secrets' do
+        let(:config) { { script: 'echo', secrets: { DATABASE_PASSWORD: { vault: 'production/db/password' } } } }
+
+        context 'when ci_secrets_syntax feature flag is enabled' do
+          before do
+            stub_feature_flags(ci_secrets_syntax: true)
+            entry.compose!
+          end
+
+          it { expect(entry).to be_valid }
+        end
+
+        context 'when ci_secrets_syntax feature flag is disabled' do
+          before do
+            stub_feature_flags(ci_secrets_syntax: false)
+            entry.compose!
+          end
+
+          it 'returns an error' do
+            aggregate_failures do
+              expect(entry).not_to be_valid
+              expect(entry.errors).to include 'job secrets feature is disabled'
+            end
+          end
+        end
+      end
     end
 
     context 'when entry value is not correct' do
+      before do
+        entry.compose!
+      end
+
       context 'when has needs' do
         context 'when needs is bridge type' do
           let(:config) do
@@ -27,6 +56,73 @@ RSpec.describe Gitlab::Ci::Config::Entry::Job do
           end
         end
       end
+
+      context 'when has invalid secrets' do
+        let(:config) { { script: 'echo', secrets: [] } }
+
+        it 'reports error' do
+          expect(entry.errors)
+            .to include 'secrets config should be a hash'
+        end
+      end
+    end
+  end
+
+  describe '.nodes' do
+    context 'when filtering all the entry/node names' do
+      subject(:nodes) { described_class.nodes }
+
+      it 'has "secrets" node' do
+        expect(nodes).to have_key(:secrets)
+      end
+    end
+  end
+
+  describe 'secrets' do
+    let(:config) { { script: 'echo', secrets: secrets } }
+    let(:secrets) do
+      {
+        DATABASE_PASSWORD: { vault: 'production/db/password' },
+        SSL_PRIVATE_KEY: { vault: 'production/ssl/private-key@ops' },
+        S3_SECRET_KEY: {
+          vault: {
+            engine: { name: 'kv-v2', path: 'aws' },
+            path: 'production/s3',
+            field: 'secret-key'
+          }
+        }
+      }
+    end
+
+    before do
+      entry.compose!
+    end
+
+    it 'includes secrets value' do
+      expect(entry.errors).to be_empty
+      expect(entry.value[:secrets]).to eq({
+        DATABASE_PASSWORD: {
+          vault: {
+            engine: { name: 'kv-v2', path: 'kv-v2' },
+            path: 'production/db',
+            field: 'password'
+          }
+        },
+        SSL_PRIVATE_KEY: {
+          vault: {
+            engine: { name: 'kv-v2', path: 'ops' },
+            path: 'production/ssl',
+            field: 'private-key'
+          }
+        },
+        S3_SECRET_KEY: {
+          vault: {
+            engine: { name: 'kv-v2', path: 'aws' },
+            path: 'production/s3',
+            field: 'secret-key'
+          }
+        }
+      })
     end
   end
 end

ee/spec/lib/gitlab/ci/config/entry/secret_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Ci::Config::Entry::Secret do
+  let(:entry) { described_class.new(config) }
+
+  describe 'validation' do
+    before do
+      entry.compose!
+    end
+
+    context 'when entry config value is correct' do
+      let(:config) do
+        {
+          vault: {
+            engine: { name: 'kv-v2', path: 'kv-v2' },
+            path: 'production/db',
+            field: 'password'
+          }
+        }
+      end
+
+      describe '#value' do
+        it 'returns secret configuration' do
+          expect(entry.value).to eq(config)
+        end
+      end
+
+      describe '#valid?' do
+        it 'is valid' do
+          expect(entry).to be_valid
+        end
+      end
+    end
+  end
+
+  context 'when entry value is not correct' do
+    describe '#errors' do
+      context 'when there is an unknown key present' do
+        let(:config) { { foo: {} } }
+
+        it 'reports error' do
+          expect(entry.errors)
+            .to include 'secret config contains unknown keys: foo'
+        end
+      end
+
+      context 'when there is no vault entry' do
+        let(:config) { {} }
+
+        it 'reports error' do
+          expect(entry.errors)
+            .to include 'secret config missing required keys: vault'
+        end
+      end
+    end
+  end
+end

ee/spec/lib/gitlab/ci/config/entry/secrets_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Ci::Config::Entry::Secrets do
+  let(:entry) { described_class.new(config) }
+
+  describe 'validation' do
+    before do
+      entry.compose!
+    end
+
+    context 'when entry config value is correct' do
+      let(:config) { {} }
+
+      describe '#value' do
+        it 'returns secrets configuration' do
+          expect(entry.value).to eq config
+        end
+      end
+
+      describe '#valid?' do
+        it 'is valid' do
+          expect(entry).to be_valid
+        end
+      end
+    end
+
+    context 'when config is of incorrect type' do
+      let(:config) { [] }
+
+      it 'reports error' do
+        expect(entry.errors)
+          .to include 'secrets config should be a hash'
+      end
+    end
+  end
+
+  describe '#compose!' do
+    context 'when valid secret entries composed' do
+      let(:config) do
+        {
+          DATABASE_PASSWORD: {
+            vault: {
+              engine: { name: 'kv-v2', path: 'kv-v2' },
+              path: 'production/db',
+              field: 'password'
+            }
+          }
+        }
+      end
+
+      before do
+        entry.compose!
+      end
+
+      describe '#value' do
+        it 'returns key value' do
+          expect(entry.value).to eq(config)
+        end
+      end
+
+      describe '#descendants' do
+        it 'creates valid descendant nodes' do
+          expect(entry.descendants).to all(be_a(Gitlab::Ci::Config::Entry::Secret))
+        end
+      end
+    end
+  end
+end

ee/spec/lib/gitlab/ci/config/entry/vault/engine_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Ci::Config::Entry::Vault::Engine do
+  let(:entry) { described_class.new(config) }
+
+  describe 'validation' do
+    before do
+      entry.compose!
+    end
+
+    context 'when entry config value is correct' do
+      let(:config) { { name: 'kv-v2', path: 'kv-v2' } }
+
+      describe '#value' do
+        it 'returns Vault secret engine configuration' do
+          expect(entry.value).to eq config
+        end
+      end
+
+      describe '#valid?' do
+        it 'is valid' do
+          expect(entry).to be_valid
+        end
+      end
+    end
+  end
+
+  context 'when entry value is not correct' do
+    describe '#errors' do
+      context 'when there is an unknown key present' do
+        let(:config) { { foo: :bar } }
+
+        it 'reports error' do
+          expect(entry.errors)
+            .to include 'engine config contains unknown keys: foo'
+        end
+      end
+
+      context 'when name and path are missing' do
+        let(:config) { {} }
+
+        it 'reports error' do
+          expect(entry.errors).to include 'engine config missing required keys: name, path'
+        end
+      end
+
+      context 'when name and path are blank' do
+        let(:config) { { name: '', path: '' } }
+
+        it 'reports error' do
+          aggregate_failures do
+            expect(entry.errors).to include 'engine name can\'t be blank'
+            expect(entry.errors).to include 'engine path can\'t be blank'
+          end
+        end
+      end
+    end
+  end
+end

ee/spec/lib/gitlab/ci/config/entry/vault/secret_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Ci::Config::Entry::Vault::Secret do
+  let(:entry) { described_class.new(config) }
+
+  describe 'validation' do
+    before do
+      entry.compose!
+    end
+
+    context 'when entry config value is correct' do
+      let(:hash_config) do
+        {
+          engine: {
+            name: 'kv-v2',
+            path: 'some/path'
+          },
+          path: 'production/db',
+          field: 'password'
+        }
+      end
+
+      context 'when config is a hash' do
+        let(:config) { hash_config }
+
+        describe '#value' do
+          it 'returns Vault secret configuration' do
+            expect(entry.value).to eq(hash_config)
+          end
+        end
+
+        describe '#valid?' do
+          it 'is valid' do
+            expect(entry).to be_valid
+          end
+        end
+      end
+
+      context 'when config is a string with engine path' do
+        let(:config) { 'production/db/password@some/path' }
+
+        describe '#value' do
+          it 'returns Vault secret configuration' do
+            expect(entry.value).to eq(hash_config)
+          end
+        end
+
+        describe '#valid?' do
+          it 'is valid' do
+            expect(entry).to be_valid
+          end
+        end
+      end
+
+      context 'when config is a string without engine path' do
+        let(:config) { 'production/db/password' }
+
+        describe '#value' do
+          it 'returns Vault secret configuration' do
+            expect(entry.value).to eq(hash_config.deep_merge(engine: { path: 'kv-v2' }))
+          end
+        end
+
+        describe '#valid?' do
+          it 'is valid' do
+            expect(entry).to be_valid
+          end
+        end
+      end
+    end
+  end
+
+  context 'when entry value is not correct' do
+    describe '#errors' do
+      context 'when there is an unknown key present' do
+        let(:config) { { foo: :bar } }
+
+        it 'reports error' do
+          expect(entry.errors)
+            .to include 'hash strategy config contains unknown keys: foo'
+        end
+      end
+
+      context 'when path is not present' do
+        let(:config) { {} }
+
+        it 'reports error' do
+          expect(entry.errors)
+            .to include 'hash strategy path can\'t be blank'
+        end
+      end
+
+      context 'when field is not present' do
+        let(:config) { {} }
+
+        it 'reports error' do
+          expect(entry.errors)
+            .to include 'hash strategy field can\'t be blank'
+        end
+      end
+
+      context 'when engine is not a hash' do
+        let(:config) { { engine: [] } }
+
+        it 'reports error' do
+          expect(entry.errors)
+            .to include 'hash strategy engine should be a hash'
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/config/entry/job.rb

@@ -15,7 +15,7 @@ module Gitlab
                             allow_failure type when start_in artifacts cache
                             dependencies before_script needs after_script
                             environment coverage retry parallel interruptible timeout
-                            resource_group release].freeze
+                            resource_group release secrets].freeze
 
           REQUIRED_BY_NEEDS = %i[stage].freeze
 
@@ -191,3 +191,5 @@ module Gitlab
     end
   end
 end
+
+::Gitlab::Ci::Config::Entry::Job.prepend_if_ee('::EE::Gitlab::Ci::Config::Entry::Job')

lib/gitlab/ci/features.rb

@@ -44,3 +44,5 @@ module Gitlab
     end
   end
 end
+
+::Gitlab::Ci::Features.prepend_if_ee('::EE::Gitlab::Ci::Features')

spec/lib/gitlab/ci/config/entry/job_spec.rb

@@ -33,7 +33,7 @@ describe Gitlab::Ci::Config::Entry::Job do
            inherit]
       end
 
-      it { is_expected.to match_array result }
+      it { is_expected.to include(*result) }
     end
   end