Merge branch 'mc/enhancement/add-external-pipeline-validation' into 'master'

Implement external validation during pipeline creation

Closes #34015

See merge request gitlab-org/gitlab!21134

app/models/ci/pipeline_enums.rb

@@ -7,7 +7,8 @@ module Ci
     def self.failure_reasons
       {
         unknown_failure: 0,
-        config_error: 1
+        config_error: 1,
+        external_validation_failure: 2
       }
     end
 

app/services/ci/create_pipeline_service.rb

@@ -16,6 +16,7 @@ module Ci
                 Gitlab::Ci::Pipeline::Chain::EvaluateWorkflowRules,
                 Gitlab::Ci::Pipeline::Chain::Seed,
                 Gitlab::Ci::Pipeline::Chain::Limit::Size,
+                Gitlab::Ci::Pipeline::Chain::Validate::External,
                 Gitlab::Ci::Pipeline::Chain::Populate,
                 Gitlab::Ci::Pipeline::Chain::Create,
                 Gitlab::Ci::Pipeline::Chain::Limit::Activity,

doc/administration/external_pipeline_validation.md

+# External Pipeline Validation
+
+You can use an external service for validating a pipeline before it's created.
+
+CAUTION: **Warning:**
+This is an experimental feature and subject to change without notice.
+
+## Usage
+
+GitLab will send a POST request to the external service URL with the pipeline
+data as payload. GitLab will then invalidate the pipeline based on the response
+code. If there's an error or the request times out, the pipeline will not be
+invalidated.
+
+Response Code Legend:
+
+- `200` - Accepted
+- `4xx` - Not Accepted
+- Other Codes - Accepted and Logged
+
+## Configuration
+
+Set the `EXTERNAL_VALIDATION_SERVICE_URL` to the external service url.
+
+## Payload Schema
+
+```json
+{
+  "type": "object",
+  "required" : [
+    "project",
+    "user",
+    "pipeline",
+    "builds"
+  ],
+  "properties" : {
+    "project": {
+      "type": "object",
+      "required": [
+        "id",
+        "path"
+      ],
+      "properties": {
+        "id": { "type": "integer" },
+        "path": { "type": "string" }
+      }
+    },
+    "user": {
+      "type": "object",
+      "required": [
+        "id",
+        "username",
+        "email"
+      ],
+      "properties": {
+        "id": { "type": "integer" },
+        "username": { "type": "string" },
+        "email": { "type": "string" }
+      }
+    },
+    "pipeline": {
+      "type": "object",
+      "required": [
+        "sha",
+        "ref",
+        "type"
+      ],
+      "properties": {
+        "sha": { "type": "string" },
+        "ref": { "type": "string" },
+        "type": { "type": "string" }
+      }
+    },
+    "builds": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": [
+          "name",
+          "stage",
+          "image",
+          "services",
+          "script"
+        ],
+        "properties": {
+          "name": { "type": "string" },
+          "stage": { "type": "string" },
+          "image": { "type": ["string", "null"] },
+          "services": {
+            "type": ["array", "null"],
+            "items": { "type": "string" }
+          },
+          "script": {
+            "type": "array",
+            "items": { "type": "string" }
+          }
+        }
+      }
+    }
+  },
+  "additionalProperties": false
+}
+```

doc/administration/index.md

@@ -152,6 +152,7 @@ Learn how to install, configure, update, and maintain your GitLab instance.
 
 - [Enable/disable GitLab CI/CD](../ci/enable_or_disable_ci.md#site-wide-admin-setting): Enable or disable GitLab CI/CD for your instance.
 - [GitLab CI/CD admin settings](../user/admin_area/settings/continuous_integration.md): Enable or disable Auto DevOps site-wide and define the artifacts' max size and expiration time.
+- [External Pipeline Validation](external_pipeline_validation.md): Enable, disable and configure external pipeline validation.
 - [Job artifacts](job_artifacts.md): Enable, disable, and configure job artifacts (a set of files and directories which are outputted by a job when it completes successfully).
 - [Job logs](job_logs.md): Information about the job logs.
 - [Register Shared and specific Runners](../ci/runners/README.md#registering-a-shared-runner): Learn how to register and configure Shared and specific Runners to your own instance.

lib/gitlab/ci/pipeline/chain/helpers.rb

@@ -5,12 +5,13 @@ module Gitlab
     module Pipeline
       module Chain
         module Helpers
-          def error(message, config_error: false)
+          def error(message, config_error: false, drop_reason: nil)
             if config_error && command.save_incompleted
+              drop_reason = :config_error
               pipeline.yaml_errors = message
-              pipeline.drop!(:config_error)
             end
 
+            pipeline.drop!(drop_reason) if drop_reason
             pipeline.errors.add(:base, message)
           end
         end

lib/gitlab/ci/pipeline/chain/validate/external.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        module Validate
+          class External < Chain::Base
+            include Chain::Helpers
+
+            InvalidResponseCode = Class.new(StandardError)
+
+            VALIDATION_REQUEST_TIMEOUT = 5
+
+            def perform!
+              error('External validation failed', drop_reason: :external_validation_failure) unless validate_external
+            end
+
+            def break?
+              @pipeline.errors.any?
+            end
+
+            private
+
+            def validate_external
+              return true unless validation_service_url
+
+              # 200 - accepted
+              # 4xx - not accepted
+              # everything else - accepted and logged
+              response_code = validate_service_request.code
+              case response_code
+              when 200
+                true
+              when 400..499
+                false
+              else
+                raise InvalidResponseCode, "Unsupported response code received from Validation Service: #{response_code}"
+              end
+            rescue => ex
+              Gitlab::Sentry.track_exception(ex)
+
+              true
+            end
+
+            def validate_service_request
+              Gitlab::HTTP.post(
+                validation_service_url, timeout: VALIDATION_REQUEST_TIMEOUT,
+                body: validation_service_payload(@pipeline, @command.config_processor.stages_attributes)
+              )
+            end
+
+            def validation_service_url
+              ENV['EXTERNAL_VALIDATION_SERVICE_URL']
+            end
+
+            def validation_service_payload(pipeline, stages_attributes)
+              {
+                project: {
+                  id: pipeline.project.id,
+                  path: pipeline.project.full_path
+                },
+                user: {
+                  id: pipeline.user.id,
+                  username: pipeline.user.username,
+                  email: pipeline.user.email
+                },
+                pipeline: {
+                  sha: pipeline.sha,
+                  ref: pipeline.ref,
+                  type: pipeline.source
+                },
+                builds: builds_validation_payload(stages_attributes)
+              }.to_json
+            end
+
+            def builds_validation_payload(stages_attributes)
+              stages_attributes.map { |stage| stage[:builds] }.flatten
+                .map(&method(:build_validation_payload))
+            end
+
+            def build_validation_payload(build)
+              {
+                name: build[:name],
+                stage: build[:stage],
+                image: build.dig(:options, :image, :name),
+                services: build.dig(:options, :services)&.map { |service| service[:name] },
+                script: [
+                  build.dig(:options, :before_script),
+                  build.dig(:options, :script),
+                  build.dig(:options, :after_script)
+                ].flatten.compact
+              }
+            end
+          end
+        end
+      end
+    end
+  end
+end

spec/fixtures/api/schemas/external_validation.json

+{
+  "type": "object",
+  "required" : [
+    "project",
+    "user",
+    "pipeline",
+    "builds"
+  ],
+  "properties" : {
+    "project": {
+      "type": "object",
+      "required": [
+        "id",
+        "path"
+      ],
+      "properties": {
+        "id": { "type": "integer" },
+        "path": { "type": "string" }
+      }
+    },
+    "user": {
+      "type": "object",
+      "required": [
+        "id",
+        "username",
+        "email"
+      ],
+      "properties": {
+        "id": { "type": "integer" },
+        "username": { "type": "string" },
+        "email": { "type": "string" }
+      }
+    },
+    "pipeline": {
+      "type": "object",
+      "required": [
+        "sha",
+        "ref",
+        "type"
+      ],
+      "properties": {
+        "sha": { "type": "string" },
+        "ref": { "type": "string" },
+        "type": { "type": "string" }
+      }
+    },
+    "builds": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": [
+          "name",
+          "stage",
+          "image",
+          "services",
+          "script"
+        ],
+        "properties": {
+          "name": { "type": "string" },
+          "stage": { "type": "string" },
+          "image": { "type": ["string", "null"] },
+          "services": {
+            "type": ["array", "null"],
+            "items": { "type": "string" }
+          },
+          "script": {
+            "type": "array",
+            "items": { "type": "string" }
+          }
+        }
+      }
+    }
+  },
+  "additionalProperties": false
+}

spec/lib/gitlab/ci/pipeline/chain/validate/external_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Ci::Pipeline::Chain::Validate::External do
+  let(:project) { create(:project) }
+  let(:user) { create(:user) }
+  let(:pipeline) { build(:ci_empty_pipeline, user: user, project: project) }
+  let!(:step) { described_class.new(pipeline, command) }
+
+  let(:ci_yaml) do
+    <<-CI_YAML
+    stages:
+      - first_stage
+      - second_stage
+
+    first_stage_job_name:
+      stage: first_stage
+      image: hello_world
+      script:
+        - echo 'hello'
+
+    second_stage_job_name:
+      stage: second_stage
+      services:
+        - postgres
+      before_script:
+        - echo 'first hello'
+      script:
+        - echo 'second hello'
+    CI_YAML
+  end
+
+  let(:yaml_processor) do
+    ::Gitlab::Ci::YamlProcessor.new(
+      ci_yaml, {
+        project: project,
+        sha: pipeline.sha,
+        user: user
+      }
+    )
+  end
+
+  let(:command) do
+    Gitlab::Ci::Pipeline::Chain::Command.new(
+      project: project, current_user: user, config_processor: yaml_processor
+    )
+  end
+
+  describe '#perform!' do
+    subject(:perform!) { step.perform! }
+
+    context 'when validation returns true' do
+      before do
+        allow(step).to receive(:validate_external).and_return(true)
+      end
+
+      it 'does not drop the pipeline' do
+        perform!
+
+        expect(pipeline.status).not_to eq('failed')
+        expect(pipeline.errors).to be_empty
+      end
+
+      it 'does not break the chain' do
+        perform!
+
+        expect(step.break?).to be false
+      end
+    end
+
+    context 'when validation return false' do
+      before do
+        allow(step).to receive(:validate_external).and_return(false)
+      end
+
+      it 'drops the pipeline' do
+        perform!
+
+        expect(pipeline.status).to eq('failed')
+        expect(pipeline.errors.to_a).to include('External validation failed')
+      end
+
+      it 'breaks the chain' do
+        perform!
+
+        expect(step.break?).to be true
+      end
+    end
+  end
+
+  describe '#validation_service_payload' do
+    subject(:validation_service_payload) { step.send(:validation_service_payload, pipeline, command.config_processor.stages_attributes) }
+
+    it 'respects the defined schema' do
+      expect(validation_service_payload).to match_schema('/external_validation')
+    end
+
+    it 'does not fire sql queries' do
+      expect { validation_service_payload }.not_to exceed_query_limit(1)
+    end
+  end
+end