Merge branch '335789-validate-against-vendored-schemas' into 'master'

Validate against vendored schemas

See merge request gitlab-org/gitlab!83478

config/feature_flags/development/enforce_security_report_validation.yml

+---
+name: enforce_security_report_validation
+introduced_by_url:
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/351000
+milestone: '14.9'
+type: development
+group: group::threat insights
+default_enabled: false

ee/spec/lib/ee/gitlab/ci/parsers/security/validators/schema_validator_spec.rb

@@ -5,17 +5,17 @@ require 'spec_helper'
 RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator do
   using RSpec::Parameterized::TableSyntax
 
-  where(:report_type, :expected_errors, :valid_data) do
-    :cluster_image_scanning     | ['root is missing required keys: vulnerabilities']                   | { 'version' => '10.0.0', 'vulnerabilities' => [] }
-    :container_scanning         | ['root is missing required keys: vulnerabilities']                   | { 'version' => '10.0.0', 'vulnerabilities' => [] }
-    :coverage_fuzzing           | ['root is missing required keys: vulnerabilities']                   | { 'version' => '10.0.0', 'vulnerabilities' => [] }
-    :dast                       | ['root is missing required keys: vulnerabilities']                   | { 'version' => '10.0.0', 'vulnerabilities' => [] }
-    :dependency_scanning        | ['root is missing required keys: dependency_files, vulnerabilities'] | { 'version' => '10.0.0', 'vulnerabilities' => [], 'dependency_files' => [] }
-    :api_fuzzing                | ['root is missing required keys: vulnerabilities']                   | { 'version' => '10.0.0', 'vulnerabilities' => [] }
+  where(:report_type, :expected_errors, :expected_warnings, :valid_data) do
+    :cluster_image_scanning     | ['root is missing required keys: vulnerabilities']                   | lazy { expected_warnings_array } | { 'version' => '10.0.0', 'vulnerabilities' => [] }
+    :container_scanning         | ['root is missing required keys: vulnerabilities']                   | lazy { expected_warnings_array } | { 'version' => '10.0.0', 'vulnerabilities' => [] }
+    :coverage_fuzzing           | ['root is missing required keys: vulnerabilities']                   | lazy { expected_warnings_array } | { 'version' => '10.0.0', 'vulnerabilities' => [] }
+    :dast                       | ['root is missing required keys: vulnerabilities']                   | lazy { expected_warnings_array } | { 'version' => '10.0.0', 'vulnerabilities' => [] }
+    :dependency_scanning        | ['root is missing required keys: dependency_files, vulnerabilities'] | lazy { expected_warnings_array } | { 'version' => '10.0.0', 'vulnerabilities' => [], 'dependency_files' => [] }
+    :api_fuzzing                | ['root is missing required keys: vulnerabilities']                   | lazy { expected_warnings_array } | { 'version' => '10.0.0', 'vulnerabilities' => [] }
   end
 
   with_them do
-    let(:validator) { described_class.new(report_type, report_data) }
+    let(:validator) { described_class.new(report_type, report_data, valid_data['version']) }
 
     describe '#valid?' do
       subject { validator.valid? }
@@ -33,6 +33,22 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator do
       end
     end
 
+    describe '#warnings' do
+      subject { validator.warnings }
+
+      context 'when given data is valid according to the schema' do
+        let(:report_data) { valid_data }
+        let(:supported_version) { described_class::SUPPORTED_VERSIONS[report_type].join(", ") }
+        let(:expected_warnings_array) do
+          [
+            "Version 10.0.0 for report type #{report_type} has been deprecated, supported versions for this report type are: #{supported_version}"
+          ]
+        end
+
+        it { is_expected.to eq(expected_warnings) }
+      end
+    end
+
     describe '#errors' do
       let(:report_data) { { 'version' => '10.0.0' } }
 

lib/gitlab/ci/parsers/security/validators/schema_validator.rb

@@ -87,19 +87,80 @@ module Gitlab
             end
 
             def initialize(report_type, report_data, report_version = nil)
-              @report_type = report_type
+              @report_type = report_type&.to_sym
               @report_data = report_data
               @report_version = report_version
+              @errors = []
+              @warnings = []
+
+              populate_errors
+              populate_warnings
             end
 
             def valid?
               errors.empty?
             end
 
-            def errors
-              @errors ||= schema.validate(report_data).map { |error| JSONSchemer::Errors.pretty(error) }
+            def populate_errors
+              if Feature.enabled?(:enforce_security_report_validation)
+                @errors += schema.validate(report_data).map { |error| JSONSchemer::Errors.pretty(error) }
+              else
+                @warnings += schema.validate(report_data).map { |error| JSONSchemer::Errors.pretty(error) }
+              end
+            end
+
+            def populate_warnings
+              add_deprecated_report_version_message if report_uses_deprecated_schema_version?
+              add_unsupported_report_version_message if !report_uses_supported_schema_version? && !report_uses_deprecated_schema_version?
+            end
+
+            def add_deprecated_report_version_message
+              message = "Version #{report_version} for report type #{report_type} has been deprecated, supported versions for this report type are: #{supported_schema_versions}"
+              add_message_as(level: :warning, message: message)
             end
 
+            def add_unsupported_report_version_message
+              if Feature.enabled?(:enforce_security_report_validation)
+                handle_unsupported_report_version(treat_as: :error)
+              else
+                handle_unsupported_report_version(treat_as: :warning)
+              end
+            end
+
+            def report_uses_deprecated_schema_version?
+              DEPRECATED_VERSIONS[report_type].include?(report_version)
+            end
+
+            def report_uses_supported_schema_version?
+              SUPPORTED_VERSIONS[report_type].include?(report_version)
+            end
+
+            def handle_unsupported_report_version(treat_as:)
+              if report_version.nil?
+                message = "Report version not provided, #{report_type} report type supports versions: #{supported_schema_versions}"
+                add_message_as(level: treat_as, message: message)
+              else
+                message = "Version #{report_version} for report type #{report_type} is unsupported, supported versions for this report type are: #{supported_schema_versions}"
+              end
+
+              add_message_as(level: treat_as, message: message)
+            end
+
+            def supported_schema_versions
+              SUPPORTED_VERSIONS[report_type].join(", ")
+            end
+
+            def add_message_as(level:, message:)
+              case level
+              when :error
+                @errors << message
+              when :warning
+                @warnings << message
+              end
+            end
+
+            attr_reader :errors, :warnings
+
             private
 
             attr_reader :report_type, :report_data, :report_version

spec/lib/gitlab/ci/parsers/security/validators/schema_validator_spec.rb

@@ -3,6 +3,8 @@
 require 'spec_helper'
 
 RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator do
+  let(:validator) { described_class.new(report_type, report_data, report_version) }
+
   describe 'SUPPORTED_VERSIONS' do
     schema_path = Rails.root.join("lib", "gitlab", "ci", "parsers", "security", "validators", "schemas")
 
@@ -47,48 +49,484 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Validators::SchemaValidator do
     end
   end
 
-  using RSpec::Parameterized::TableSyntax
+  describe '#valid?' do
+    subject { validator.valid? }
 
-  where(:report_type, :report_version, :expected_errors, :valid_data) do
-    'sast' | '10.0.0' | ['root is missing required keys: vulnerabilities'] | { 'version' => '10.0.0', 'vulnerabilities' => [] }
-    :sast  | '10.0.0' | ['root is missing required keys: vulnerabilities'] | { 'version' => '10.0.0', 'vulnerabilities' => [] }
-    :secret_detection | '10.0.0' | ['root is missing required keys: vulnerabilities'] | { 'version' => '10.0.0', 'vulnerabilities' => [] }
-  end
+    context 'when given a supported schema version' do
+      let(:report_type) { :dast }
+      let(:report_version) { described_class::SUPPORTED_VERSIONS[report_type].last }
 
-  with_them do
-    let(:validator) { described_class.new(report_type, report_data, report_version) }
+      context 'and the report is valid' do
+        let(:report_data) do
+          {
+            'version' => report_version,
+            'vulnerabilities' => []
+          }
+        end
 
-    describe '#valid?' do
-      subject { validator.valid? }
+        it { is_expected.to be_truthy }
+      end
 
-      context 'when given data is invalid according to the schema' do
-        let(:report_data) { {} }
+      context 'and the report is invalid' do
+        let(:report_data) do
+          {
+            'version' => report_version
+          }
+        end
 
         it { is_expected.to be_falsey }
       end
+    end
+
+    context 'when given a deprecated schema version' do
+      let(:report_type) { :dast }
+      let(:report_version) { described_class::DEPRECATED_VERSIONS[report_type].last }
 
-      context 'when given data is valid according to the schema' do
-        let(:report_data) { valid_data }
+      context 'and the report passes schema validation' do
+        let(:report_data) do
+          {
+            'version' => '10.0.0',
+            'vulnerabilities' => []
+          }
+        end
 
         it { is_expected.to be_truthy }
       end
 
-      context 'when no report_version is provided' do
-        let(:report_version) { nil }
-        let(:report_data) { valid_data }
+      context 'and the report does not pass schema validation' do
+        context 'and enforce_security_report_validation is enabled' do
+          before do
+            stub_feature_flags(enforce_security_report_validation: true)
+          end
+
+          let(:report_data) do
+            {
+              'version' => 'V2.7.0'
+            }
+          end
+
+          it { is_expected.to be_falsey }
+        end
+
+        context 'and enforce_security_report_validation is disabled' do
+          before do
+            stub_feature_flags(enforce_security_report_validation: false)
+          end
+
+          let(:report_data) do
+            {
+              'version' => 'V2.7.0'
+            }
+          end
+
+          it { is_expected.to be_truthy }
+        end
+      end
+    end
+
+    context 'when given an unsupported schema version' do
+      let(:report_type) { :dast }
+      let(:report_version) { "12.37.0" }
+
+      context 'if enforce_security_report_validation is enabled' do
+        before do
+          stub_feature_flags(enforce_security_report_validation: true)
+        end
+
+        context 'and the report is valid' do
+          let(:report_data) do
+            {
+              'version' => report_version,
+              'vulnerabilities' => []
+            }
+          end
+
+          it { is_expected.to be_falsey }
+        end
+
+        context 'and the report is invalid' do
+          let(:report_data) do
+            {
+              'version' => report_version
+            }
+          end
+
+          it { is_expected.to be_falsey }
+        end
+      end
+
+      context 'if enforce_security_report_validation is disabled' do
+        before do
+          stub_feature_flags(enforce_security_report_validation: false)
+        end
+
+        context 'and the report is valid' do
+          let(:report_data) do
+            {
+              'version' => report_version,
+              'vulnerabilities' => []
+            }
+          end
+
+          it { is_expected.to be_truthy }
+        end
+
+        context 'and the report is invalid' do
+          let(:report_data) do
+            {
+              'version' => report_version
+            }
+          end
+
+          it { is_expected.to be_truthy }
+        end
+      end
+    end
+  end
+
+  describe '#errors' do
+    subject { validator.errors }
+
+    context 'when given a supported schema version' do
+      let(:report_type) { :dast }
+      let(:report_version) { described_class::SUPPORTED_VERSIONS[report_type].last }
+
+      context 'and the report is valid' do
+        let(:report_data) do
+          {
+            'version' => report_version,
+            'vulnerabilities' => []
+          }
+        end
+
+        let(:expected_errors) { [] }
+
+        it { is_expected.to match_array(expected_errors) }
+      end
+
+      context 'and the report is invalid' do
+        let(:report_data) do
+          {
+            'version' => report_version
+          }
+        end
+
+        let(:expected_errors) do
+          [
+            'root is missing required keys: vulnerabilities'
+          ]
+        end
+
+        it { is_expected.to match_array(expected_errors) }
+      end
+    end
+
+    context 'when given a deprecated schema version' do
+      let(:report_type) { :dast }
+      let(:report_version) { described_class::DEPRECATED_VERSIONS[report_type].last }
+
+      context 'and the report passes schema validation' do
+        let(:report_data) do
+          {
+            'version' => '10.0.0',
+            'vulnerabilities' => []
+          }
+        end
+
+        let(:expected_errors) { [] }
+
+        it { is_expected.to match_array(expected_errors) }
+      end
+
+      context 'and the report does not pass schema validation' do
+        context 'and enforce_security_report_validation is enabled' do
+          before do
+            stub_feature_flags(enforce_security_report_validation: true)
+          end
+
+          let(:report_data) do
+            {
+              'version' => 'V2.7.0'
+            }
+          end
+
+          let(:expected_errors) do
+            [
+              "property '/version' does not match pattern: ^[0-9]+\\.[0-9]+\\.[0-9]+$",
+              "root is missing required keys: vulnerabilities"
+            ]
+          end
+
+          it { is_expected.to match_array(expected_errors) }
+        end
+
+        context 'and enforce_security_report_validation is disabled' do
+          before do
+            stub_feature_flags(enforce_security_report_validation: false)
+          end
+
+          let(:report_data) do
+            {
+              'version' => 'V2.7.0'
+            }
+          end
+
+          let(:expected_errors) { [] }
 
-        it 'does not fail' do
-          expect { subject }.not_to raise_error
+          it { is_expected.to match_array(expected_errors) }
         end
       end
     end
 
-    describe '#errors' do
-      let(:report_data) { { 'version' => '10.0.0' } }
+    context 'when given an unsupported schema version' do
+      let(:report_type) { :dast }
+      let(:report_version) { "12.37.0" }
 
-      subject { validator.errors }
+      context 'if enforce_security_report_validation is enabled' do
+        before do
+          stub_feature_flags(enforce_security_report_validation: true)
+        end
+
+        context 'and the report is valid' do
+          let(:report_data) do
+            {
+              'version' => report_version,
+              'vulnerabilities' => []
+            }
+          end
 
-      it { is_expected.to eq(expected_errors) }
+          let(:expected_errors) do
+            [
+              "Version 12.37.0 for report type dast is unsupported, supported versions for this report type are: 14.0.0, 14.0.1, 14.0.2, 14.0.3, 14.0.4, 14.0.5, 14.0.6, 14.1.0"
+            ]
+          end
+
+          it { is_expected.to match_array(expected_errors) }
+        end
+
+        context 'and the report is invalid' do
+          let(:report_data) do
+            {
+              'version' => report_version
+            }
+          end
+
+          let(:expected_errors) do
+            [
+              "Version 12.37.0 for report type dast is unsupported, supported versions for this report type are: 14.0.0, 14.0.1, 14.0.2, 14.0.3, 14.0.4, 14.0.5, 14.0.6, 14.1.0",
+              "root is missing required keys: vulnerabilities"
+            ]
+          end
+
+          it { is_expected.to match_array(expected_errors) }
+        end
+      end
+
+      context 'if enforce_security_report_validation is disabled' do
+        before do
+          stub_feature_flags(enforce_security_report_validation: false)
+        end
+
+        context 'and the report is valid' do
+          let(:report_data) do
+            {
+              'version' => report_version,
+              'vulnerabilities' => []
+            }
+          end
+
+          let(:expected_errors) { [] }
+
+          it { is_expected.to match_array(expected_errors) }
+        end
+
+        context 'and the report is invalid' do
+          let(:report_data) do
+            {
+              'version' => report_version
+            }
+          end
+
+          let(:expected_errors) { [] }
+
+          it { is_expected.to match_array(expected_errors) }
+        end
+      end
+    end
+  end
+
+  describe '#warnings' do
+    subject { validator.warnings }
+
+    context 'when given a supported schema version' do
+      let(:report_type) { :dast }
+      let(:report_version) { described_class::SUPPORTED_VERSIONS[report_type].last }
+
+      context 'and the report is valid' do
+        let(:report_data) do
+          {
+            'version' => report_version,
+            'vulnerabilities' => []
+          }
+        end
+
+        let(:expected_warnings) { [] }
+
+        it { is_expected.to match_array(expected_warnings) }
+      end
+
+      context 'and the report is invalid' do
+        let(:report_data) do
+          {
+            'version' => report_version
+          }
+        end
+
+        let(:expected_warnings) { [] }
+
+        it { is_expected.to match_array(expected_warnings) }
+      end
+    end
+
+    context 'when given a deprecated schema version' do
+      let(:report_type) { :dast }
+      let(:report_version) { described_class::DEPRECATED_VERSIONS[report_type].last }
+
+      context 'and the report passes schema validation' do
+        let(:report_data) do
+          {
+            'vulnerabilities' => []
+          }
+        end
+
+        let(:expected_warnings) do
+          [
+            "Version V2.7.0 for report type dast has been deprecated, supported versions for this report type are: 14.0.0, 14.0.1, 14.0.2, 14.0.3, 14.0.4, 14.0.5, 14.0.6, 14.1.0"
+          ]
+        end
+
+        it { is_expected.to match_array(expected_warnings) }
+      end
+
+      context 'and the report does not pass schema validation' do
+        context 'and enforce_security_report_validation is enabled' do
+          before do
+            stub_feature_flags(enforce_security_report_validation: true)
+          end
+
+          let(:report_data) do
+            {
+              'version' => 'V2.7.0'
+            }
+          end
+
+          let(:expected_warnings) do
+            [
+              "Version V2.7.0 for report type dast has been deprecated, supported versions for this report type are: 14.0.0, 14.0.1, 14.0.2, 14.0.3, 14.0.4, 14.0.5, 14.0.6, 14.1.0"
+            ]
+          end
+
+          it { is_expected.to match_array(expected_warnings) }
+        end
+
+        context 'and enforce_security_report_validation is disabled' do
+          before do
+            stub_feature_flags(enforce_security_report_validation: false)
+          end
+
+          let(:report_data) do
+            {
+              'version' => 'V2.7.0'
+            }
+          end
+
+          let(:expected_warnings) do
+            [
+              "Version V2.7.0 for report type dast has been deprecated, supported versions for this report type are: 14.0.0, 14.0.1, 14.0.2, 14.0.3, 14.0.4, 14.0.5, 14.0.6, 14.1.0",
+              "property '/version' does not match pattern: ^[0-9]+\\.[0-9]+\\.[0-9]+$",
+              "root is missing required keys: vulnerabilities"
+            ]
+          end
+
+          it { is_expected.to match_array(expected_warnings) }
+        end
+      end
+    end
+
+    context 'when given an unsupported schema version' do
+      let(:report_type) { :dast }
+      let(:report_version) { "12.37.0" }
+
+      context 'if enforce_security_report_validation is enabled' do
+        before do
+          stub_feature_flags(enforce_security_report_validation: true)
+        end
+
+        context 'and the report is valid' do
+          let(:report_data) do
+            {
+              'version' => report_version,
+              'vulnerabilities' => []
+            }
+          end
+
+          let(:expected_warnings) { [] }
+
+          it { is_expected.to match_array(expected_warnings) }
+        end
+
+        context 'and the report is invalid' do
+          let(:report_data) do
+            {
+              'version' => report_version
+            }
+          end
+
+          let(:expected_warnings) { [] }
+
+          it { is_expected.to match_array(expected_warnings) }
+        end
+      end
+
+      context 'if enforce_security_report_validation is disabled' do
+        before do
+          stub_feature_flags(enforce_security_report_validation: false)
+        end
+
+        context 'and the report is valid' do
+          let(:report_data) do
+            {
+              'version' => report_version,
+              'vulnerabilities' => []
+            }
+          end
+
+          let(:expected_warnings) do
+            [
+              "Version 12.37.0 for report type dast is unsupported, supported versions for this report type are: 14.0.0, 14.0.1, 14.0.2, 14.0.3, 14.0.4, 14.0.5, 14.0.6, 14.1.0"
+            ]
+          end
+
+          it { is_expected.to match_array(expected_warnings) }
+        end
+
+        context 'and the report is invalid' do
+          let(:report_data) do
+            {
+              'version' => report_version
+            }
+          end
+
+          let(:expected_warnings) do
+            [
+              "Version 12.37.0 for report type dast is unsupported, supported versions for this report type are: 14.0.0, 14.0.1, 14.0.2, 14.0.3, 14.0.4, 14.0.5, 14.0.6, 14.1.0",
+              "root is missing required keys: vulnerabilities"
+            ]
+          end
+
+          it { is_expected.to match_array(expected_warnings) }
+        end
+      end
     end
   end
 end