Merge branch '299215-add-ci-configuration-processor' into 'master'

Add new processor to CI Configuration with Security Policies [RUN ALL RSPEC] [RUN AS-IF-FOSS]

See merge request gitlab-org/gitlab!54419

ee/lib/ee/gitlab/ci/config_ee.rb

@@ -15,12 +15,18 @@ module EE
 
         override :build_config
         def build_config(config)
-          process_required_includes(super)
+          super
+            .then(&method(:process_required_includes))
+            .then(&method(:process_security_orchestration_policy_includes))
         end
 
         def process_required_includes(config)
           ::Gitlab::Ci::Config::Required::Processor.new(config).perform
         end
+
+        def process_security_orchestration_policy_includes(config)
+          ::Gitlab::Ci::Config::SecurityOrchestrationPolicies::Processor.new(config, context.project, ref).perform
+        end
       end
     end
   end

ee/lib/gitlab/ci/config/security_orchestration_policies/processor.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module SecurityOrchestrationPolicies
+        class Processor
+          def initialize(config, project, ref)
+            @config = config
+            @project = project
+            @ref = ref
+          end
+
+          def perform
+            return @config unless project&.feature_available?(:security_orchestration_policies)
+            return @config unless security_orchestration_policy_configuration&.enabled?
+
+            @config
+              .deep_merge(on_demand_scans_template)
+          end
+
+          def on_demand_scans_template
+            ::Security::SecurityOrchestrationPolicies::OnDemandScanPipelineConfigurationService
+              .new(project)
+              .execute(security_orchestration_policy_configuration.on_demand_scan_actions(@ref))
+          end
+
+          private
+
+          attr_reader :project
+
+          delegate :security_orchestration_policy_configuration, to: :project, allow_nil: true
+        end
+      end
+    end
+  end
+end

ee/spec/lib/ee/gitlab/ci/config_spec.rb

@@ -3,33 +3,152 @@
 require 'spec_helper'
 
 RSpec.describe Gitlab::Ci::Config do
-  let(:template_name) { 'test_template' }
-  let(:template_repository) { create(:project, :custom_repo, files: { "gitlab-ci/#{template_name}.yml" => template_yml }) }
-
-  let(:ci_yml) do
+  let_it_be(:ci_yml) do
     <<-EOS
     sample_job:
       script:
-        - echo 'test'
+      - echo 'test'
     EOS
   end
 
-  let(:template_yml) do
-    <<-EOS
-    sample_job:
-      script:
-        - echo 'not test'
-    EOS
-  end
+  describe 'with required instance template' do
+    let(:template_name) { 'test_template' }
+    let(:template_repository) { create(:project, :custom_repo, files: { "gitlab-ci/#{template_name}.yml" => template_yml }) }
 
-  subject { described_class.new(ci_yml) }
+    let(:template_yml) do
+      <<-EOS
+      sample_job:
+        script:
+          - echo 'not test'
+      EOS
+    end
 
-  before do
-    stub_application_setting(file_template_project: template_repository, required_instance_ci_template: template_name)
-    stub_licensed_features(custom_file_templates: true, required_ci_templates: true)
+    subject(:config) { described_class.new(ci_yml) }
+
+    before do
+      stub_application_setting(file_template_project: template_repository, required_instance_ci_template: template_name)
+      stub_licensed_features(custom_file_templates: true, required_ci_templates: true)
+    end
+
+    it 'processes the required includes' do
+      expect(config.to_hash[:sample_job][:script]).to eq(["echo 'not test'"])
+    end
   end
 
-  it 'processes the required includes' do
-    expect(subject.to_hash[:sample_job][:script]).to eq(["echo 'not test'"])
+  describe 'with security orchestration policy' do
+    let_it_be(:ref) { 'master' }
+    let_it_be_with_refind(:project) { create(:project, :repository) }
+
+    let_it_be(:policies_repository) { create(:project, :repository) }
+    let_it_be(:security_orchestration_policy_configuration) { create(:security_orchestration_policy_configuration, project: project, security_policy_management_project: policies_repository) }
+
+    let_it_be(:policy_yml) do
+      <<-EOS
+      scan_execution_policy:
+       -  name: Run DAST in every pipeline
+          description: This policy enforces to run DAST for every pipeline within the project
+          enabled: true
+          rules:
+          - type: pipeline
+            branches:
+            - "production"
+          actions:
+          - scan: dast
+            site_profile: Site Profile
+            scanner_profile: Scanner Profile
+      EOS
+    end
+
+    subject(:config) { described_class.new(ci_yml, ref: ref, project: project) }
+
+    before do
+      allow_next_instance_of(Repository) do |repository|
+        # allow(repository).to receive(:ls_files).and_return(['.gitlab/security-policies/enforce-dast.yml'])
+        allow(repository).to receive(:blob_data_at).and_return(policy_yml)
+      end
+    end
+
+    context 'when feature is not licensed' do
+      it 'does not modify the config' do
+        expect(config.to_hash).to eq(sample_job: { script: ["echo 'test'"] })
+      end
+    end
+
+    context 'when feature is licensed' do
+      before do
+        stub_licensed_features(security_orchestration_policies: true)
+      end
+
+      context 'when feature is not enabled' do
+        before do
+          stub_feature_flags(security_orchestration_policies_configuration: false)
+        end
+
+        it 'does not modify the config' do
+          expect(config.to_hash).to eq(sample_job: { script: ["echo 'test'"] })
+        end
+      end
+
+      context 'when feature is enabled' do
+        before do
+          stub_feature_flags(security_orchestration_policies_configuration: true)
+        end
+
+        context 'when policy is not applicable on branch from the pipeline' do
+          it 'does not modify the config' do
+            expect(config.to_hash).to eq(sample_job: { script: ["echo 'test'"] })
+          end
+        end
+
+        context 'when policy is not applicable on branch from the pipeline' do
+          let_it_be(:ref) { 'production' }
+
+          context 'when DAST profiles are not found' do
+            it 'adds a job with error message' do
+              expect(config.to_hash).to eq(
+                sample_job: { script: ["echo 'test'"] },
+                'dast-on-demand-0': { allow_failure: true, script: 'echo "Error during On-Demand Scan execution: Dast site profile was not provided" && false' }
+              )
+            end
+          end
+
+          context 'when DAST profiles are found' do
+            let_it_be(:dast_scanner_profile) { create(:dast_scanner_profile, project: project, name: 'Scanner Profile') }
+            let_it_be(:dast_site_profile) { create(:dast_site_profile, project: project, name: 'Site Profile') }
+
+            let(:expected_configuration) do
+              {
+                sample_job: {
+                  script: ["echo 'test'"]
+                },
+                'dast-on-demand-0': {
+                  stage: 'test',
+                  image: { name: '$SECURE_ANALYZERS_PREFIX/dast:$DAST_VERSION' },
+                  variables: {
+                    DAST_VERSION: 1,
+                    SECURE_ANALYZERS_PREFIX: 'registry.gitlab.com/gitlab-org/security-products/analyzers',
+                    DAST_WEBSITE: dast_site_profile.dast_site.url,
+                    DAST_FULL_SCAN_ENABLED: 'false',
+                    DAST_USE_AJAX_SPIDER: 'false',
+                    DAST_DEBUG: 'false',
+                    DAST_USERNAME:  dast_site_profile.auth_username,
+                    DAST_EXCLUDE_URLS: dast_site_profile.excluded_urls.join(','),
+                    DAST_USERNAME_FIELD: 'session[username]',
+                    DAST_PASSWORD_FIELD: 'session[password]'
+               },
+                  allow_failure: true,
+                  script: ['/analyze'],
+                  artifacts: { reports: { dast: 'gl-dast-report.json' } }
+                }
+              }
+            end
+
+            it 'extends config with additional jobs' do
+              expect(config.to_hash).to include(expected_configuration)
+            end
+          end
+        end
+      end
+    end
   end
 end

ee/spec/lib/gitlab/ci/config/security_orchestration_policies/processor_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Ci::Config::SecurityOrchestrationPolicies::Processor do
+  subject { described_class.new(config, project, ref).perform }
+
+  let_it_be(:config) { { image: 'ruby:2.5.3' } }
+
+  let_it_be(:ref) { 'master' }
+  let_it_be_with_refind(:project) { create(:project, :repository) }
+
+  let_it_be(:policies_repository) { create(:project, :repository) }
+  let_it_be(:security_orchestration_policy_configuration) { create(:security_orchestration_policy_configuration, project: project, security_policy_management_project: policies_repository) }
+
+  let_it_be(:policy_yml) do
+    <<-EOS
+    scan_execution_policy:
+      -  name: Run DAST in every pipeline
+         description: This policy enforces to run DAST for every pipeline within the project
+         enabled: true
+         rules:
+         - type: pipeline
+           branches:
+           - "production"
+         actions:
+         - scan: dast
+           site_profile: Site Profile
+           scanner_profile: Scanner Profile
+    EOS
+  end
+
+  before do
+    allow_next_instance_of(Repository) do |repository|
+      allow(repository).to receive(:blob_data_at).and_return(policy_yml)
+    end
+  end
+
+  context 'when feature is not licensed' do
+    it 'does not modify the config' do
+      expect(subject).to eq(config)
+    end
+  end
+
+  context 'when feature is licensed' do
+    before do
+      stub_licensed_features(security_orchestration_policies: true)
+    end
+
+    context 'when feature is not enabled' do
+      before do
+        stub_feature_flags(security_orchestration_policies_configuration: false)
+      end
+
+      it 'does not modify the config' do
+        expect(subject).to eq(config)
+      end
+    end
+
+    context 'when feature is enabled' do
+      before do
+        stub_feature_flags(security_orchestration_policies_configuration: true)
+      end
+
+      context 'when policy is not applicable on branch from the pipeline' do
+        it 'does not modify the config' do
+          expect(subject).to eq(config)
+        end
+      end
+
+      context 'when policy is not applicable on branch from the pipeline' do
+        let_it_be(:ref) { 'production' }
+
+        context 'when DAST profiles are not found' do
+          it 'does not modify the config' do
+            expect(subject).to eq(
+              image: 'ruby:2.5.3',
+              'dast-on-demand-0': { allow_failure: true, script: 'echo "Error during On-Demand Scan execution: Dast site profile was not provided" && false' }
+            )
+          end
+        end
+
+        context 'when DAST profiles are found' do
+          let_it_be(:dast_scanner_profile) { create(:dast_scanner_profile, project: project, name: 'Scanner Profile') }
+          let_it_be(:dast_site_profile) { create(:dast_site_profile, project: project, name: 'Site Profile') }
+
+          let(:expected_configuration) do
+            {
+              image: 'ruby:2.5.3',
+              'dast-on-demand-0': {
+                stage: 'test',
+                image: {
+                  name: '$SECURE_ANALYZERS_PREFIX/dast:$DAST_VERSION'
+                },
+                variables: {
+                  DAST_VERSION: 1,
+                  SECURE_ANALYZERS_PREFIX: 'registry.gitlab.com/gitlab-org/security-products/analyzers',
+                  DAST_WEBSITE: dast_site_profile.dast_site.url,
+                  DAST_FULL_SCAN_ENABLED: 'false',
+                  DAST_USE_AJAX_SPIDER: 'false',
+                  DAST_DEBUG: 'false',
+                  DAST_USERNAME:  dast_site_profile.auth_username,
+                  DAST_EXCLUDE_URLS: dast_site_profile.excluded_urls.join(','),
+                  DAST_USERNAME_FIELD: 'session[username]',
+                  DAST_PASSWORD_FIELD: 'session[password]'
+                },
+                allow_failure: true,
+                script: ['/analyze'],
+                artifacts: {
+                  reports: {
+                    dast: 'gl-dast-report.json'
+                  }
+                }
+              }
+            }
+          end
+
+          it 'extends config with additional jobs' do
+            expect(subject).to include(expected_configuration)
+          end
+        end
+      end
+    end
+  end
+end

ee/spec/models/security/orchestration_policy_configuration_spec.rb

@@ -29,6 +29,10 @@ RSpec.describe Security::OrchestrationPolicyConfiguration do
     subject { security_orchestration_policy_configuration.enabled? }
 
     context 'when feature is enabled' do
+      before do
+        stub_feature_flags(security_orchestration_policies_configuration: true)
+      end
+
       it { is_expected.to eq(true) }
     end
 

lib/gitlab/ci/config.rb

@@ -17,12 +17,14 @@ module Gitlab
         Config::Yaml::Tags::TagError
       ].freeze
 
-      attr_reader :root
+      attr_reader :root, :context, :ref
 
-      def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil)
+      def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil, ref: nil)
         @context = build_context(project: project, sha: sha, user: user, parent_pipeline: parent_pipeline)
         @context.set_deadline(TIMEOUT_SECONDS)
 
+        @ref = ref
+
         @config = expand_config(config)
 
         @root = Entry::Root.new(@config)

lib/gitlab/ci/pipeline/chain/config/process.rb

@@ -14,6 +14,7 @@ module Gitlab
               result = ::Gitlab::Ci::YamlProcessor.new(
                 @command.config_content, {
                   project: project,
+                  ref: @pipeline.ref,
                   sha: @pipeline.sha,
                   user: current_user,
                   parent_pipeline: parent_pipeline