Merge branch '343489-fix-analyzers-prefix-apisec' into 'master'

Support SECURE_ANALYZERS_PREFIX variable in API Security templates

See merge request gitlab-org/gitlab!73876

config/metrics/counts_28d/20210216184559_ci_templates_total_unique_counts_monthly.yml

@@ -49,6 +49,7 @@ options:
   - p_ci_templates_security_api_fuzzing_latest
   - p_ci_templates_security_secure_binaries
   - p_ci_templates_security_dast_api
+  - p_ci_templates_security_dast_api_latest
   - p_ci_templates_security_container_scanning
   - p_ci_templates_security_dast_latest
   - p_ci_templates_security_dependency_scanning
@@ -153,6 +154,7 @@ options:
   - p_ci_templates_implicit_security_api_fuzzing_latest
   - p_ci_templates_implicit_security_secure_binaries
   - p_ci_templates_implicit_security_dast_api
+  - p_ci_templates_implicit_security_dast_api_latest
   - p_ci_templates_implicit_security_container_scanning
   - p_ci_templates_implicit_security_dast_latest
   - p_ci_templates_implicit_security_dependency_scanning

config/metrics/counts_28d/20211122134101_p_ci_templates_implicit_security_dast_api_latest_monthly.yml

+---
+key_path: redis_hll_counters.ci_templates.p_ci_templates_implicit_security_dast_api_latest_monthly
+description: Monthly counts for DAST API latest CI template
+product_section: sec
+product_stage: secure
+product_group: dynamic_analysis
+product_category: dynamic_application_security_testing
+value_type: number
+status: active
+milestone: '14.6'
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/73876
+time_frame: 28d
+data_source: redis_hll
+data_category: optional
+instrumentation_class: RedisHLLMetric
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate
+options:
+  events:
+  - p_ci_templates_implicit_security_dast_api_latest

config/metrics/counts_28d/20211201194407_p_ci_templates_security_dast_api_latest_monthly.yml

+---
+key_path: redis_hll_counters.ci_templates.p_ci_templates_security_dast_api_latest_monthly
+description: Monthly counts for DAST API latest CI template
+product_section: sec
+product_stage: secure
+product_group: dynamic_analysis
+product_category: dynamic_application_security_testing
+value_type: number
+status: active
+milestone: "14.6"
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/73876
+time_frame: 28d
+data_source: redis_hll
+data_category: optional
+instrumentation_class: RedisHLLMetric
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate
+options:
+  events:
+  - p_ci_templates_security_dast_api_latest

config/metrics/counts_7d/20210216184557_ci_templates_total_unique_counts_weekly.yml

@@ -49,6 +49,7 @@ options:
   - p_ci_templates_security_api_fuzzing_latest
   - p_ci_templates_security_secure_binaries
   - p_ci_templates_security_dast_api
+  - p_ci_templates_security_dast_api_latest
   - p_ci_templates_security_container_scanning
   - p_ci_templates_security_dast_latest
   - p_ci_templates_security_dependency_scanning
@@ -153,6 +154,7 @@ options:
   - p_ci_templates_implicit_security_api_fuzzing_latest
   - p_ci_templates_implicit_security_secure_binaries
   - p_ci_templates_implicit_security_dast_api
+  - p_ci_templates_implicit_security_dast_api_latest
   - p_ci_templates_implicit_security_container_scanning
   - p_ci_templates_implicit_security_dast_latest
   - p_ci_templates_implicit_security_dependency_scanning

config/metrics/counts_7d/20211122134101_p_ci_templates_implicit_security_dast_api_latest_weekly.yml

+---
+key_path: redis_hll_counters.ci_templates.p_ci_templates_implicit_security_dast_api_latest_weekly
+description: Weekly counts for DAST API latest CI template
+product_section: sec
+product_stage: secure
+product_group: dynamic_analysis
+product_category: dynamic_application_security_testing
+value_type: number
+status: active
+milestone: '14.6'
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/73876
+time_frame: 7d
+data_source: redis_hll
+data_category: optional
+instrumentation_class: RedisHLLMetric
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate
+options:
+  events:
+  - p_ci_templates_implicit_security_dast_api_latest

config/metrics/counts_7d/20211201194402_p_ci_templates_security_dast_api_latest_weekly.yml

+---
+key_path: redis_hll_counters.ci_templates.p_ci_templates_security_dast_api_latest_weekly
+description: Weekly counts for DAST API latest CI template
+product_section: sec
+product_stage: secure
+product_group: dynamic_analysis
+product_category: dynamic_application_security_testing
+value_type: number
+status: active
+milestone: "14.6"
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/73876
+time_frame: 7d
+data_source: redis_hll
+data_category: optional
+instrumentation_class: RedisHLLMetric
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate
+options:
+  events:
+  - p_ci_templates_security_dast_api_latest

ee/spec/lib/gitlab/ci/templates/api_fuzzing_latest_gitlab_ci_yaml_spec.rb

@@ -10,8 +10,8 @@ RSpec.describe 'API-Fuzzing.latest.gitlab-ci.yml' do
   describe 'the template file' do
     let(:template_filename) { Rails.root.join("lib/gitlab/ci/templates/" + template.full_name) }
     let(:contents) { File.read(template_filename) }
-    let(:production_registry) { '${SECURE_ANALYZERS_PREFIX}/api-fuzzing:${FUZZAPI_VERSION}' }
-    let(:staging_registry) { '${SECURE_ANALYZERS_PREFIX}/api-fuzzing-src:${FUZZAPI_VERSION}' }
+    let(:production_registry) { 'FUZZAPI_IMAGE: api-fuzzing' }
+    let(:staging_registry) { 'FUZZAPI_IMAGE: api-fuzzing-src' }
 
     # Make sure future changes to the template use the production container registry.
     #

ee/spec/lib/gitlab/ci/templates/dast_api_latest_gitlab_ci_yaml_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe 'DAST-API.latest.gitlab-ci.yml' do
+  subject(:template) { Gitlab::Template::GitlabCiYmlTemplate.find('DAST-API.latest') }
+
+  specify { expect(template).not_to be_nil }
+
+  describe 'the template file' do
+    let(:template_filename) { Rails.root.join("lib/gitlab/ci/templates/" + template.full_name) }
+    let(:contents) { File.read(template_filename) }
+    let(:production_registry) { 'DAST_API_IMAGE: api-fuzzing' }
+    let(:staging_registry) { 'DAST_API_IMAGE: api-fuzzing-src' }
+
+    # Make sure future changes to the template use the production container registry.
+    #
+    # The DAST API template is developed against a dev container registry.
+    # The registry is switched when releasing new versions. The difference in
+    # names between development and production is also quite small making it
+    # easy to miss during review.
+    it 'uses the production repository' do
+      expect(contents.include?(production_registry)).to be true
+    end
+
+    it "doesn't use the staging repository" do
+      expect(contents.include?(staging_registry)).to be false
+    end
+  end
+
+  describe 'the created pipeline' do
+    let(:default_branch) { 'master' }
+    let(:pipeline_branch) { default_branch }
+    let_it_be(:project) { create(:project, :custom_repo, files: { 'README.txt' => '' }) }
+
+    let(:user) { project.owner }
+    let(:service) { Ci::CreatePipelineService.new(project, user, ref: pipeline_branch ) }
+    let(:pipeline) { service.execute!(:push).payload }
+    let(:build_names) { pipeline.builds.pluck(:name) }
+
+    before do
+      allow_next_instance_of(Ci::BuildScheduleWorker) do |worker|
+        allow(worker).to receive(:perform).and_return(true)
+      end
+
+      allow(project).to receive(:default_branch).and_return(default_branch)
+    end
+
+    context 'when no stages' do
+      before do
+        stub_ci_pipeline_yaml_file(template.content)
+      end
+
+      context 'when project has no stages' do
+        it 'includes no jobs' do
+          expect(build_names).to be_empty
+        end
+      end
+    end
+
+    context 'when stages includes dast' do
+      let(:ci_pipeline_yaml) { "stages: [\"dast\"]\n" }
+
+      before do
+        stub_ci_pipeline_yaml_file(ci_pipeline_yaml + template.content)
+      end
+
+      context 'when project has no license' do
+        before do
+          create(:ci_variable, project: project, key: 'DAST_API_HAR', value: 'testing.har')
+          create(:ci_variable, project: project, key: 'DAST_API_TARGET_URL', value: 'http://example.com')
+        end
+
+        it 'includes job to display error' do
+          expect(build_names).to match_array(%w[dast_api])
+        end
+      end
+
+      context 'when project has Ultimate license' do
+        before do
+          stub_licensed_features(dast: true)
+        end
+
+        context 'by default' do
+          it 'includes a job' do
+            expect(build_names).to match_array(%w[dast_api])
+          end
+        end
+
+        context 'when DAST_API_DISABLED=1' do
+          before do
+            create(:ci_variable, project: project, key: 'DAST_API_DISABLED', value: '1')
+            create(:ci_variable, project: project, key: 'DAST_API_HAR', value: 'testing.har')
+            create(:ci_variable, project: project, key: 'DAST_API_TARGET_URL', value: 'http://example.com')
+          end
+
+          it 'includes no jobs' do
+            expect { pipeline }.to raise_error(Ci::CreatePipelineService::CreateError)
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml

@@ -11,11 +11,11 @@
 variables:
     FUZZAPI_VERSION: "1"
     SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers"
-    FUZZAPI_IMAGE: ${SECURE_ANALYZERS_PREFIX}/api-fuzzing:${FUZZAPI_VERSION}
+    FUZZAPI_IMAGE: api-fuzzing
 
 apifuzzer_fuzz:
     stage: fuzz
-    image: $FUZZAPI_IMAGE
+    image: $SECURE_ANALYZERS_PREFIX/$FUZZAPI_IMAGE:$FUZZAPI_VERSION
     allow_failure: true
     rules:
         - if: $API_FUZZING_DISABLED

lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml

+# To contribute improvements to CI/CD templates, please follow the Development guide at:
+# https://docs.gitlab.com/ee/development/cicd/templates.html
+# This specific template is located at:
+# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Dast-API.gitlab-ci.yml
+
+# To use this template, add the following to your .gitlab-ci.yml file:
+#
+# include:
+#   template: DAST-API.latest.gitlab-ci.yml
+#
+# You also need to add a `dast` stage to your `stages:` configuration. A sample configuration for DAST API:
+#
+# stages:
+#   - build
+#   - test
+#   - deploy
+#   - dast
+
+# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast_api/index.html
+
+# Configure DAST API scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html).
+# List of available variables: https://docs.gitlab.com/ee/user/application_security/dast_api/index.html#available-cicd-variables
+
+variables:
+  # Setting this variable affects all Security templates
+  # (SAST, Dependency Scanning, ...)
+  SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers"
+  #
+  DAST_API_VERSION: "1"
+  DAST_API_IMAGE: api-fuzzing
+
+dast_api:
+  stage: dast
+  image: $SECURE_ANALYZERS_PREFIX/$DAST_API_IMAGE:$DAST_API_VERSION
+  allow_failure: true
+  rules:
+    - if: $DAST_API_DISABLED
+      when: never
+    - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH &&
+          $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
+      when: never
+    - if: $CI_COMMIT_BRANCH
+  script:
+    - /peach/analyzer-dast-api
+  artifacts:
+    when: always
+    paths:
+      - gl-assets
+      - gl-dast-api-report.json
+      - gl-*.log
+    reports:
+      dast: gl-dast-api-report.json

lib/gitlab/usage_data_counters/known_events/ci_templates.yml

@@ -119,6 +119,10 @@
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
+- name: p_ci_templates_security_dast_api_latest
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
 - name: p_ci_templates_security_container_scanning
   category: ci_templates
   redis_slot: ci_templates
@@ -551,6 +555,10 @@
   category: ci_templates
   redis_slot: ci_templates
   aggregation: weekly
+- name: p_ci_templates_implicit_security_dast_api_latest
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
 - name: p_ci_templates_implicit_security_container_scanning
   category: ci_templates
   redis_slot: ci_templates