Merge branch '331631-apifuzzing-update-ci' into 'master'

Promote unstable API Fuzzing CI template to stable (breaking)

See merge request gitlab-org/gitlab!62407

ee/spec/lib/ee/gitlab/ci/templates/templates_spec.rb

@@ -4,12 +4,9 @@ require 'spec_helper'
 
 RSpec.describe "CI YML Templates" do
   using RSpec::Parameterized::TableSyntax
-
   subject { Gitlab::Ci::YamlProcessor.new(content).execute }
 
-  where(:template_name) do
-    Gitlab::Template::GitlabCiYmlTemplate.all.map(&:full_name)
-  end
+  let(:all_templates) { Gitlab::Template::GitlabCiYmlTemplate.all.map(&:full_name) }
 
   before do
     stub_feature_flags(
@@ -19,27 +16,24 @@ RSpec.describe "CI YML Templates" do
       redirect_to_latest_template_jobs_browser_performance_testing: false)
   end
 
-  with_them do
-    let(:content) do
-      if template_name == 'Security/DAST-API.gitlab-ci.yml'
-        # The DAST-API template purposly excludes a stages
-        # definition.
+  shared_examples 'require default stages to be included' do
+    it 'require default stages to be included' do
+      expect(subject.stages).to include(*Gitlab::Ci::Config::Entry::Stages.default)
+    end
+  end
 
-        <<~EOS
-          include:
-            - template: #{template_name}
+  context 'that support autodevops' do
+    non_autodevops_templates = [
+      'Security/DAST-API.gitlab-ci.yml',
+      'Security/API-Fuzzing.gitlab-ci.yml'
+    ]
 
-          stages:
-            - build
-            - test
-            - deploy
-            - dast
+    where(:template_name) do
+      all_templates - non_autodevops_templates
+    end
 
-          concrete_build_implemented_by_a_user:
-            stage: test
-            script: do something
-        EOS
-      else
+    with_them do
+      let(:content) do
         <<~EOS
           include:
             - template: #{template_name}
@@ -49,14 +43,104 @@ RSpec.describe "CI YML Templates" do
             script: do something
         EOS
       end
+
+      it 'are valid with default stages' do
+        expect(subject).to be_valid
+      end
+
+      include_examples 'require default stages to be included'
     end
+  end
+
+  context 'that do not support autodevops' do
+    context 'when DAST API template' do
+      # The DAST API template purposly excludes a stages
+      # definition.
+
+      let(:template_name) { 'Security/DAST-API.gitlab-ci.yml' }
+
+      context 'with default stages' do
+        let(:content) do
+          <<~EOS
+            include:
+              - template: #{template_name}
 
-    it 'is valid' do
-      expect(subject).to be_valid
+            concrete_build_implemented_by_a_user:
+              stage: test
+              script: do something
+          EOS
+        end
+
+        it { is_expected.not_to be_valid }
+      end
+
+      context 'with defined stages' do
+        let(:content) do
+          <<~EOS
+            include:
+              - template: #{template_name}
+
+            stages:
+              - build
+              - test
+              - deploy
+              - dast
+
+            concrete_build_implemented_by_a_user:
+              stage: test
+              script: do something
+          EOS
+        end
+
+        it { is_expected.to be_valid }
+
+        include_examples 'require default stages to be included'
+      end
     end
 
-    it 'require default stages to be included' do
-      expect(subject.stages).to include(*Gitlab::Ci::Config::Entry::Stages.default)
+    context 'when API Fuzzing template' do
+      # The API Fuzzing template purposly excludes a stages
+      # definition.
+
+      let(:template_name) { 'Security/API-Fuzzing.gitlab-ci.yml' }
+
+      context 'with default stages' do
+        let(:content) do
+          <<~EOS
+            include:
+              - template: #{template_name}
+
+            concrete_build_implemented_by_a_user:
+              stage: test
+              script: do something
+          EOS
+        end
+
+        it { is_expected.not_to be_valid }
+      end
+
+      context 'with defined stages' do
+        let(:content) do
+          <<~EOS
+            include:
+              - template: #{template_name}
+
+            stages:
+              - build
+              - test
+              - deploy
+              - fuzz
+
+            concrete_build_implemented_by_a_user:
+              stage: test
+              script: do something
+          EOS
+        end
+
+        it { is_expected.to be_valid }
+
+        include_examples 'require default stages to be included'
+      end
     end
   end
 end

ee/spec/lib/gitlab/ci/templates/api_fuzzing_gitlab_ci_yaml_spec.rb

@@ -5,6 +5,8 @@ require 'spec_helper'
 RSpec.describe 'API-Fuzzing.gitlab-ci.yml' do
   subject(:template) { Gitlab::Template::GitlabCiYmlTemplate.find('API-Fuzzing') }
 
+  specify { expect(template).not_to be_nil }
+
   describe 'the template file' do
     let(:template_filename) { Rails.root.join("lib/gitlab/ci/templates/" + template.full_name) }
     let(:contents) { File.read(template_filename) }
@@ -36,106 +38,103 @@ RSpec.describe 'API-Fuzzing.gitlab-ci.yml' do
     let(:pipeline) { service.execute!(:push) }
     let(:build_names) { pipeline.builds.pluck(:name) }
 
-    before do
-      stub_ci_pipeline_yaml_file(template.content)
-      allow_next_instance_of(Ci::BuildScheduleWorker) do |worker|
-        allow(worker).to receive(:perform).and_return(true)
-      end
-      allow(project).to receive(:default_branch).and_return(default_branch)
-    end
-
-    context 'when project has no license' do
+    context 'when no stages' do
       before do
-        create(:ci_variable, project: project, key: 'FUZZAPI_HAR', value: 'testing.har')
-        create(:ci_variable, project: project, key: 'FUZZAPI_TARGET_URL', value: 'http://example.com')
+        stub_ci_pipeline_yaml_file(template.content)
+        allow_next_instance_of(Ci::BuildScheduleWorker) do |worker|
+          allow(worker).to receive(:perform).and_return(true)
+        end
+        allow(project).to receive(:default_branch).and_return(default_branch)
       end
 
-      it 'includes job to display error' do
-        expect(build_names).to match_array(%w[apifuzzer_fuzz_unlicensed])
+      context 'when project has no stages' do
+        it 'includes no jobs' do
+          expect(build_names).to be_empty
+        end
       end
     end
 
-    context 'when project has Ultimate license' do
-      let(:license) { build(:license, plan: License::ULTIMATE_PLAN) }
+    context 'when stages includes fuzz' do
+      let(:ci_pipeline_yaml) { "stages: [\"fuzz\"]\n" }
 
       before do
-        allow(License).to receive(:current).and_return(license)
-      end
+        stub_ci_pipeline_yaml_file(ci_pipeline_yaml + template.content)
 
-      context 'by default' do
-        it 'includes a job' do
-          expect(build_names).to match_array(%w[apifuzzer_fuzz])
+        allow_next_instance_of(Ci::BuildScheduleWorker) do |worker|
+          allow(worker).to receive(:perform).and_return(true)
         end
+
+        allow(project).to receive(:default_branch).and_return(default_branch)
       end
 
-      context 'when configured with HAR' do
+      context 'when project has no license' do
         before do
           create(:ci_variable, project: project, key: 'FUZZAPI_HAR', value: 'testing.har')
           create(:ci_variable, project: project, key: 'FUZZAPI_TARGET_URL', value: 'http://example.com')
         end
 
-        it 'includes job' do
+        it 'includes job to display error' do
           expect(build_names).to match_array(%w[apifuzzer_fuzz])
         end
       end
 
-      context 'when configured with OpenAPI' do
+      context 'when project has Ultimate license' do
+        let(:license) { build(:license, plan: License::ULTIMATE_PLAN) }
+
         before do
-          create(:ci_variable, project: project, key: 'FUZZAPI_OPENAPI', value: 'testing.json')
-          create(:ci_variable, project: project, key: 'FUZZAPI_TARGET_URL', value: 'http://example.com')
+          allow(License).to receive(:current).and_return(license)
         end
 
-        it 'includes job' do
-          expect(build_names).to match_array(%w[apifuzzer_fuzz])
+        context 'by default' do
+          it 'includes a job' do
+            expect(build_names).to match_array(%w[apifuzzer_fuzz])
+          end
         end
-      end
 
-      context 'when configured with Postman' do
-        before do
-          create(:ci_variable, project: project, key: 'FUZZAPI_POSTMAN_COLLECTION', value: 'testing.json')
-          create(:ci_variable, project: project, key: 'FUZZAPI_TARGET_URL', value: 'http://example.com')
-        end
+        context 'when configured with HAR' do
+          before do
+            create(:ci_variable, project: project, key: 'FUZZAPI_HAR', value: 'testing.har')
+            create(:ci_variable, project: project, key: 'FUZZAPI_TARGET_URL', value: 'http://example.com')
+          end
 
-        it 'includes job' do
-          expect(build_names).to match_array(%w[apifuzzer_fuzz])
+          it 'includes job' do
+            expect(build_names).to match_array(%w[apifuzzer_fuzz])
+          end
         end
-      end
 
-      context 'when FUZZAPI_D_TARGET_IMAGE is present' do
-        before do
-          create(:ci_variable, project: project, key: 'FUZZAPI_D_TARGET_IMAGE', value: 'imagename:latest')
-          create(:ci_variable, project: project, key: 'FUZZAPI_HAR', value: 'testing.har')
-          create(:ci_variable, project: project, key: 'FUZZAPI_TARGET_URL', value: 'http://example.com')
-        end
+        context 'when configured with OpenAPI' do
+          before do
+            create(:ci_variable, project: project, key: 'FUZZAPI_OPENAPI', value: 'testing.json')
+            create(:ci_variable, project: project, key: 'FUZZAPI_TARGET_URL', value: 'http://example.com')
+          end
 
-        it 'includes dnd job' do
-          expect(build_names).to match_array(%w[apifuzzer_fuzz_dnd])
+          it 'includes job' do
+            expect(build_names).to match_array(%w[apifuzzer_fuzz])
+          end
         end
-      end
-    end
 
-    context 'when API_FUZZING_DISABLED=1' do
-      before do
-        create(:ci_variable, project: project, key: 'API_FUZZING_DISABLED', value: '1')
-        create(:ci_variable, project: project, key: 'FUZZAPI_HAR', value: 'testing.har')
-        create(:ci_variable, project: project, key: 'FUZZAPI_TARGET_URL', value: 'http://example.com')
-      end
+        context 'when configured with Postman' do
+          before do
+            create(:ci_variable, project: project, key: 'FUZZAPI_POSTMAN_COLLECTION', value: 'testing.json')
+            create(:ci_variable, project: project, key: 'FUZZAPI_TARGET_URL', value: 'http://example.com')
+          end
 
-      it 'includes no jobs' do
-        expect { pipeline }.to raise_error(Ci::CreatePipelineService::CreateError)
-      end
-    end
+          it 'includes job' do
+            expect(build_names).to match_array(%w[apifuzzer_fuzz])
+          end
+        end
 
-    context 'when API_FUZZING_DISABLED=1 with DnD' do
-      before do
-        create(:ci_variable, project: project, key: 'API_FUZZING_DISABLED', value: '1')
-        create(:ci_variable, project: project, key: 'FUZZAPI_D_TARGET_IMAGE', value: 'imagename:latest')
-        create(:ci_variable, project: project, key: 'FUZZAPI_HAR', value: 'testing.har')
-        create(:ci_variable, project: project, key: 'FUZZAPI_TARGET_URL', value: 'http://example.com')
-      end
+        context 'when API_FUZZING_DISABLED=1' do
+          before do
+            create(:ci_variable, project: project, key: 'API_FUZZING_DISABLED', value: '1')
+            create(:ci_variable, project: project, key: 'FUZZAPI_HAR', value: 'testing.har')
+            create(:ci_variable, project: project, key: 'FUZZAPI_TARGET_URL', value: 'http://example.com')
+          end
 
-      it 'includes no jobs' do
-        expect { pipeline }.to raise_error(Ci::CreatePipelineService::CreateError)
+          it 'includes no jobs' do
+            expect { pipeline }.to raise_error(Ci::CreatePipelineService::CreateError)
+          end
+        end
       end
     end
   end

lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml

@@ -3,276 +3,31 @@
 # Configure API fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html).
 # List of available variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-cicd-variables
 
-stages:
-    - build
-    - test
-    - deploy
-    - fuzz
-
 variables:
+    FUZZAPI_VERSION: "1"
     SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers"
-    FUZZAPI_PROFILE: Quick
-    FUZZAPI_VERSION: "1.6"
-    FUZZAPI_CONFIG: .gitlab-api-fuzzing.yml
-    FUZZAPI_TIMEOUT: 30
-    FUZZAPI_REPORT: gl-api-fuzzing-report.json
-    FUZZAPI_REPORT_ASSET_PATH: assets
-    #
-    FUZZAPI_D_NETWORK: testing-net
-    #
-    # Wait up to 5 minutes for API Fuzzer and target url to become
-    # available (non 500 response to HTTP(s))
-    FUZZAPI_SERVICE_START_TIMEOUT: "300"
-    #
     FUZZAPI_IMAGE: ${SECURE_ANALYZERS_PREFIX}/api-fuzzing:${FUZZAPI_VERSION}
-    #
-
-apifuzzer_fuzz_unlicensed:
-    stage: fuzz
-    allow_failure: true
-    rules:
-        - if: '$GITLAB_FEATURES !~ /\bapi_fuzzing\b/ && $API_FUZZING_DISABLED == null'
-        - when: never
-    script:
-        - |
-            echo "Error: Your GitLab project is not licensed for API Fuzzing."
-        - exit 1
 
 apifuzzer_fuzz:
     stage: fuzz
-    image:
-        name: $FUZZAPI_IMAGE
-        entrypoint: ["/bin/bash", "-l", "-c"]
-    variables:
-        FUZZAPI_PROJECT: $CI_PROJECT_PATH
-        FUZZAPI_API: http://localhost:5000
-        FUZZAPI_NEW_REPORT: 1
-        FUZZAPI_LOG_SCANNER: gl-apifuzzing-api-scanner.log
-        TZ: America/Los_Angeles
+    image: $FUZZAPI_IMAGE
     allow_failure: true
     rules:
-        - if: $FUZZAPI_D_TARGET_IMAGE
-          when: never
-        - if: $FUZZAPI_D_WORKER_IMAGE
-          when: never
-        - if: $API_FUZZING_DISABLED
-          when: never
-        - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH &&
-                $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
-          when: never
-        - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/
-    script:
-        #
-        # Validate options
-        - |
-            if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \
-                echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \
-                echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \
-                exit 1; \
-            fi
-        #
-        # Run user provided pre-script
-        - sh -c "$FUZZAPI_PRE_SCRIPT"
-        #
-        # Make sure asset path exists
-        - mkdir -p $FUZZAPI_REPORT_ASSET_PATH
-        #
-        # Start API Security background process
-        - dotnet /peach/Peach.Web.dll &> $FUZZAPI_LOG_SCANNER &
-        - APISEC_PID=$!
-        #
-        # Start scanning
-        - worker-entry
-        #
-        # Run user provided post-script
-        - sh -c "$FUZZAPI_POST_SCRIPT"
-        #
-        # Shutdown API Security
-        - kill $APISEC_PID
-        - wait $APISEC_PID
-        #
-    artifacts:
-        when: always
-        paths:
-            - $FUZZAPI_REPORT_ASSET_PATH
-            - $FUZZAPI_REPORT
-            - $FUZZAPI_LOG_SCANNER
-        reports:
-            api_fuzzing: $FUZZAPI_REPORT
-
-apifuzzer_fuzz_dnd:
-    stage: fuzz
-    image: docker:19.03.12
-    variables:
-        DOCKER_DRIVER: overlay2
-        DOCKER_TLS_CERTDIR: ""
-        FUZZAPI_PROJECT: $CI_PROJECT_PATH
-        FUZZAPI_API: http://apifuzzer:5000
-    allow_failure: true
-    rules:
-        - if: $FUZZAPI_D_TARGET_IMAGE == null && $FUZZAPI_D_WORKER_IMAGE == null
-          when: never
         - if: $API_FUZZING_DISABLED
           when: never
         - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH &&
                 $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
           when: never
-        - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/
-    services:
-        - docker:19.03.12-dind
+        - if: $CI_COMMIT_BRANCH
     script:
-        #
-        #
-        - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
-        #
-        - docker network create --driver bridge $FUZZAPI_D_NETWORK
-        #
-        # Run user provided pre-script
-        - sh -c "$FUZZAPI_PRE_SCRIPT"
-        #
-        # Make sure asset path exists
-        - mkdir -p $FUZZAPI_REPORT_ASSET_PATH
-        #
-        # Start peach testing engine container
-        - |
-            docker run -d \
-            --name apifuzzer \
-            --network $FUZZAPI_D_NETWORK \
-            -e Proxy:Port=8000 \
-            -e TZ=America/Los_Angeles \
-            -e GITLAB_FEATURES \
-            -p 80:80 \
-            -p 5000:5000 \
-            -p 8000:8000 \
-            -p 514:514 \
-            --restart=no \
-            $FUZZAPI_IMAGE \
-            dotnet /peach/Peach.Web.dll
-        #
-        # Start target container
-        - |
-            if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then \
-                docker run -d \
-                    --name target \
-                    --network $FUZZAPI_D_NETWORK \
-                    $FUZZAPI_D_TARGET_ENV \
-                    $FUZZAPI_D_TARGET_PORTS \
-                    $FUZZAPI_D_TARGET_VOLUME \
-                    --restart=no \
-                    $FUZZAPI_D_TARGET_IMAGE \
-                ; fi
-        #
-        # Start worker container if provided
-        - |
-            if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then \
-                echo "Starting worker image $FUZZAPI_D_WORKER_IMAGE"; \
-                docker run \
-                    --name worker \
-                    --network $FUZZAPI_D_NETWORK \
-                    -e FUZZAPI_API=http://apifuzzer:5000 \
-                    -e FUZZAPI_PROJECT \
-                    -e FUZZAPI_PROFILE \
-                    -e FUZZAPI_CONFIG \
-                    -e FUZZAPI_REPORT \
-                    -e FUZZAPI_REPORT_ASSET_PATH \
-                    -e FUZZAPI_NEW_REPORT=1 \
-                    -e FUZZAPI_HAR \
-                    -e FUZZAPI_OPENAPI \
-                    -e FUZZAPI_POSTMAN_COLLECTION \
-                    -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \
-                    -e FUZZAPI_TARGET_URL \
-                    -e FUZZAPI_OVERRIDES_FILE \
-                    -e FUZZAPI_OVERRIDES_ENV \
-                    -e FUZZAPI_OVERRIDES_CMD \
-                    -e FUZZAPI_OVERRIDES_INTERVAL \
-                    -e FUZZAPI_TIMEOUT \
-                    -e FUZZAPI_VERBOSE \
-                    -e FUZZAPI_SERVICE_START_TIMEOUT \
-                    -e FUZZAPI_HTTP_USERNAME \
-                    -e FUZZAPI_HTTP_PASSWORD \
-                    -e CI_PROJECT_URL \
-                    -e CI_JOB_ID \
-                    -e CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH} \
-                    $FUZZAPI_D_WORKER_ENV \
-                    $FUZZAPI_D_WORKER_PORTS \
-                    $FUZZAPI_D_WORKER_VOLUME \
-                    --restart=no \
-                    $FUZZAPI_D_WORKER_IMAGE \
-                ; fi
-        #
-        # Start API Fuzzing provided worker if no other worker present
-        - |
-            if [ "$FUZZAPI_D_WORKER_IMAGE" == "" ]; then \
-                if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \
-                    echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \
-                    echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \
-                    exit 1; \
-                fi; \
-                docker run \
-                    --name worker \
-                    --network $FUZZAPI_D_NETWORK \
-                    -e TZ=America/Los_Angeles \
-                    -e FUZZAPI_API=http://apifuzzer:5000 \
-                    -e FUZZAPI_PROJECT \
-                    -e FUZZAPI_PROFILE \
-                    -e FUZZAPI_CONFIG \
-                    -e FUZZAPI_REPORT \
-                    -e FUZZAPI_REPORT_ASSET_PATH \
-                    -e FUZZAPI_NEW_REPORT=1 \
-                    -e FUZZAPI_HAR \
-                    -e FUZZAPI_OPENAPI \
-                    -e FUZZAPI_POSTMAN_COLLECTION \
-                    -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \
-                    -e FUZZAPI_TARGET_URL \
-                    -e FUZZAPI_OVERRIDES_FILE \
-                    -e FUZZAPI_OVERRIDES_ENV \
-                    -e FUZZAPI_OVERRIDES_CMD \
-                    -e FUZZAPI_OVERRIDES_INTERVAL \
-                    -e FUZZAPI_TIMEOUT \
-                    -e FUZZAPI_VERBOSE \
-                    -e FUZZAPI_SERVICE_START_TIMEOUT \
-                    -e FUZZAPI_HTTP_USERNAME \
-                    -e FUZZAPI_HTTP_PASSWORD \
-                    -e CI_PROJECT_URL \
-                    -e CI_JOB_ID \
-                    -v $CI_PROJECT_DIR:/app \
-                    -v `pwd`/$FUZZAPI_REPORT_ASSET_PATH:/app/$FUZZAPI_REPORT_ASSET_PATH:rw \
-                    -p 81:80 \
-                    -p 5001:5000 \
-                    -p 8001:8000 \
-                    -p 515:514 \
-                    --restart=no \
-                    $FUZZAPI_IMAGE \
-                    worker-entry \
-                ; fi
-        #
-        # Propagate exit code from api fuzzing scanner (if any)
-        - if [[ $(docker inspect apifuzzer --format='{{.State.ExitCode}}') != "0" ]]; then echo "API Fuzzing scanner exited with an error. Logs are available as job artifacts."; exit 1; fi
-        #
-        # Run user provided post-script
-        - sh -c "$FUZZAPI_POST_SCRIPT"
-        #
-    after_script:
-        #
-        # Shutdown all containers
-        - echo "Stopping all containers"
-        - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker stop target; fi
-        - docker stop worker
-        - docker stop apifuzzer
-        #
-        # Save docker logs
-        - docker logs apifuzzer &> gl-api_fuzzing-logs.log
-        - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker logs target &> gl-api_fuzzing-target-logs.log; fi
-        - docker logs worker &> gl-api_fuzzing-worker-logs.log
-        #
+        - /peach/analyzer-fuzz-api
     artifacts:
         when: always
         paths:
-            - ./gl-api_fuzzing*.log
-            - ./gl-api_fuzzing*.zip
-            - $FUZZAPI_REPORT_ASSET_PATH
-            - $FUZZAPI_REPORT
+            - gl-assets
+            - gl-api-fuzzing-report.json
+            - gl-*.log
         reports:
-            api_fuzzing: $FUZZAPI_REPORT
+            api_fuzzing: gl-api-fuzzing-report.json
 
 # end

spec/lib/gitlab/ci/templates/templates_spec.rb

@@ -6,33 +6,104 @@ RSpec.describe 'CI YML Templates' do
   subject { Gitlab::Ci::YamlProcessor.new(content).execute }
 
   let(:all_templates) { Gitlab::Template::GitlabCiYmlTemplate.all.map(&:full_name) }
-
   let(:excluded_templates) do
     all_templates.select do |name|
       Gitlab::Template::GitlabCiYmlTemplate.excluded_patterns.any? { |pattern| pattern.match?(name) }
     end
   end
 
-  context 'when including available templates in a CI YAML configuration' do
-    using RSpec::Parameterized::TableSyntax
+  before do
+    stub_feature_flags(
+      redirect_to_latest_template_terraform: false,
+      redirect_to_latest_template_security_api_fuzzing: false,
+      redirect_to_latest_template_security_dast: false)
+  end
+
+  shared_examples 'require default stages to be included' do
+    it 'require default stages to be included' do
+      expect(subject.stages).to include(*Gitlab::Ci::Config::Entry::Stages.default)
+    end
+  end
+
+  context 'that support autodevops' do
+    non_autodevops_templates = [
+      'Security/DAST-API.gitlab-ci.yml',
+      'Security/API-Fuzzing.gitlab-ci.yml'
+    ]
+
+    context 'when including available templates in a CI YAML configuration' do
+      using RSpec::Parameterized::TableSyntax
+
+      where(:template_name) do
+        all_templates - excluded_templates - non_autodevops_templates
+      end
+
+      with_them do
+        let(:content) do
+          <<~EOS
+            include:
+              - template: #{template_name}
+
+            concrete_build_implemented_by_a_user:
+              stage: test
+              script: do something
+          EOS
+        end
+
+        it { is_expected.to be_valid }
 
-    where(:template_name) do
-      all_templates - excluded_templates
+        include_examples 'require default stages to be included'
+      end
     end
 
-    before do
-      stub_feature_flags(
-        redirect_to_latest_template_terraform: false,
-        redirect_to_latest_template_security_api_fuzzing: false,
-        redirect_to_latest_template_security_dast: false)
+    context 'when including unavailable templates in a CI YAML configuration' do
+      using RSpec::Parameterized::TableSyntax
+
+      where(:template_name) do
+        excluded_templates
+      end
+
+      with_them do
+        let(:content) do
+          <<~EOS
+            include:
+              - template: #{template_name}
+
+            concrete_build_implemented_by_a_user:
+              stage: test
+              script: do something
+          EOS
+        end
+
+        it { is_expected.not_to be_valid }
+      end
     end
+  end
+
+  describe 'that do not support autodevops' do
+    context 'when DAST API template' do
+      # The DAST API template purposly excludes a stages
+      # definition.
+
+      let(:template_name) { 'Security/DAST-API.gitlab-ci.yml' }
 
-    with_them do
-      let(:content) do
-        if template_name == 'Security/DAST-API.gitlab-ci.yml'
-          # The DAST-API template purposly excludes a stages
-          # definition.
+      context 'with default stages' do
+        let(:content) do
+          <<~EOS
+            include:
+              - template: #{template_name}
+
+            concrete_build_implemented_by_a_user:
+              stage: test
+              script: do something
+          EOS
+        end
 
+        it { is_expected.not_to be_valid }
+      end
+
+      context 'with defined stages' do
+        let(:content) do
           <<~EOS
             include:
               - template: #{template_name}
@@ -47,7 +118,22 @@ RSpec.describe 'CI YML Templates' do
               stage: test
               script: do something
           EOS
-        else
+        end
+
+        it { is_expected.to be_valid }
+
+        include_examples 'require default stages to be included'
+      end
+    end
+
+    context 'when API Fuzzing template' do
+      # The API Fuzzing template purposly excludes a stages
+      # definition.
+
+      let(:template_name) { 'Security/API-Fuzzing.gitlab-ci.yml' }
+
+      context 'with default stages' do
+        let(:content) do
           <<~EOS
             include:
               - template: #{template_name}
@@ -57,39 +143,31 @@ RSpec.describe 'CI YML Templates' do
               script: do something
           EOS
         end
-      end
-
-      it 'is valid' do
-        expect(subject).to be_valid
-      end
 
-      it 'require default stages to be included' do
-        expect(subject.stages).to include(*Gitlab::Ci::Config::Entry::Stages.default)
+        it { is_expected.not_to be_valid }
       end
-    end
-  end
 
-  context 'when including unavailable templates in a CI YAML configuration' do
-    using RSpec::Parameterized::TableSyntax
+      context 'with defined stages' do
+        let(:content) do
+          <<~EOS
+            include:
+              - template: #{template_name}
 
-    where(:template_name) do
-      excluded_templates
-    end
+            stages:
+              - build
+              - test
+              - deploy
+              - fuzz
 
-    with_them do
-      let(:content) do
-        <<~EOS
-          include:
-            - template: #{template_name}
+            concrete_build_implemented_by_a_user:
+              stage: test
+              script: do something
+          EOS
+        end
 
-          concrete_build_implemented_by_a_user:
-            stage: test
-            script: do something
-        EOS
-      end
+        it { is_expected.to be_valid }
 
-      it 'is not valid' do
-        expect(subject).not_to be_valid
+        include_examples 'require default stages to be included'
       end
     end
   end