Merge branch '326146-add-dependency-scanning-to-adoption-table' into 'master'

[DevopsAdoption] Add dependency scanning to Devops Adoption API

See merge request gitlab-org/gitlab!64624

db/migrate/20210621091830_add_devops_adoption_snapshot_dependency_scanning.rb

+# frozen_string_literal: true
+
+class AddDevopsAdoptionSnapshotDependencyScanning < ActiveRecord::Migration[6.1]
+  def change
+    add_column :analytics_devops_adoption_snapshots, :dependency_scanning_enabled_count, :integer
+  end
+end

db/post_migrate/20210621111747_add_ci_artifacts_devops_adoption_index.rb

+# frozen_string_literal: true
+#
+class AddCiArtifactsDevopsAdoptionIndex < ActiveRecord::Migration[6.1]
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  NEW_INDEX = 'index_ci_job_artifacts_on_file_type_for_devops_adoption'
+
+  def up
+    add_concurrent_index :ci_job_artifacts, [:file_type, :project_id, :created_at], name: NEW_INDEX, where: 'file_type IN (5,6,8,23)'
+  end
+
+  def down
+    remove_concurrent_index_by_name :ci_job_artifacts, NEW_INDEX
+  end
+end

db/schema_migrations/20210621091830

+11450695142e5145e5cbf9077f259e28c908b794948be5c4b77e34465f6498aa
\ No newline at end of file

db/schema_migrations/20210621111747

+582a22626330e93e679a14eff2cdd85a5732f797fc1a6c94409d46cc5027c8ea
\ No newline at end of file

db/structure.sql

@@ -9128,6 +9128,7 @@ CREATE TABLE analytics_devops_adoption_snapshots (
     namespace_id integer,
     sast_enabled_count integer,
     dast_enabled_count integer,
+    dependency_scanning_enabled_count integer,
     CONSTRAINT check_3f472de131 CHECK ((namespace_id IS NOT NULL))
 );
 
@@ -22935,6 +22936,8 @@ CREATE INDEX index_ci_job_artifacts_on_expire_at_and_job_id ON ci_job_artifacts
 
 CREATE INDEX index_ci_job_artifacts_on_file_store ON ci_job_artifacts USING btree (file_store);
 
+CREATE INDEX index_ci_job_artifacts_on_file_type_for_devops_adoption ON ci_job_artifacts USING btree (file_type, project_id, created_at) WHERE (file_type = ANY (ARRAY[5, 6, 8, 23]));
+
 CREATE UNIQUE INDEX index_ci_job_artifacts_on_job_id_and_file_type ON ci_job_artifacts USING btree (job_id, file_type);
 
 CREATE INDEX index_ci_job_artifacts_on_project_id ON ci_job_artifacts USING btree (project_id);

doc/api/graphql/reference/index.md

@@ -8507,6 +8507,7 @@ Snapshot.
 | ---- | ---- | ----------- |
 | <a id="devopsadoptionsnapshotcodeownersusedcount"></a>`codeOwnersUsedCount` | [`Int`](#int) | Total number of projects with existing CODEOWNERS file. |
 | <a id="devopsadoptionsnapshotdastenabledcount"></a>`dastEnabledCount` | [`Int`](#int) | Total number of projects with enabled DAST. |
+| <a id="devopsadoptionsnapshotdependencyscanningenabledcount"></a>`dependencyScanningEnabledCount` | [`Int`](#int) | Total number of projects with enabled dependency scanning. |
 | <a id="devopsadoptionsnapshotdeploysucceeded"></a>`deploySucceeded` | [`Boolean!`](#boolean) | At least one deployment succeeded. |
 | <a id="devopsadoptionsnapshotendtime"></a>`endTime` | [`Time!`](#time) | The end time for the snapshot where the data points were collected. |
 | <a id="devopsadoptionsnapshotissueopened"></a>`issueOpened` | [`Boolean!`](#boolean) | At least one issue was opened. |

ee/app/graphql/types/analytics/devops_adoption/snapshot_type.rb

@@ -28,6 +28,8 @@ module Types
               description: 'Total number of projects with enabled SAST.'
         field :dast_enabled_count, GraphQL::INT_TYPE, null: true,
               description: 'Total number of projects with enabled DAST.'
+        field :dependency_scanning_enabled_count, GraphQL::INT_TYPE, null: true,
+              description: 'Total number of projects with enabled dependency scanning.'
         field :total_projects_count, GraphQL::INT_TYPE, null: true,
               description: 'Total number of projects.'
         field :recorded_at, Types::TimeType, null: false,

ee/app/models/analytics/devops_adoption/snapshot.rb

@@ -17,6 +17,7 @@ class Analytics::DevopsAdoption::Snapshot < ApplicationRecord
     :code_owners_used_count,
     :sast_enabled_count,
     :dast_enabled_count,
+    :dependency_scanning_enabled_count,
     :total_projects_count
   ].freeze
 

ee/config/feature_flags/development/analytics_devops_adoption_dependency_scanning.yml

+---
+name: analytics_devops_adoption_dependency_scanning
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/64624
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/335083
+milestone: '14.1'
+type: development
+group: group::optimize
+default_enabled: false

ee/lib/analytics/devops_adoption/snapshot_calculator.rb

@@ -95,25 +95,32 @@ module Analytics
         end
       end
 
-      # rubocop: disable CodeReuse/ActiveRecord
       def sast_enabled_count
         return unless Feature.enabled?(:analytics_devops_adoption_sastdast, enabled_namespace.namespace, default_enabled: :yaml)
 
-        Ci::JobArtifact.sast_reports
-                       .for_project(snapshot_project_ids)
-                       .created_in_time_range(from: range_start, to: range_end)
-                       .select(:project_id).distinct.count
+        projects_count_with_artifact(Ci::JobArtifact.sast_reports)
       end
-      # rubocop: enable CodeReuse/ActiveRecord
 
-      # rubocop: disable CodeReuse/ActiveRecord
       def dast_enabled_count
         return unless Feature.enabled?(:analytics_devops_adoption_sastdast, enabled_namespace.namespace, default_enabled: :yaml)
 
-        Ci::JobArtifact.dast_reports
-          .for_project(snapshot_project_ids)
-          .created_in_time_range(from: range_start, to: range_end)
-          .select(:project_id).distinct.count
+        projects_count_with_artifact(Ci::JobArtifact.dast_reports)
+      end
+
+      def dependency_scanning_enabled_count
+        return unless Feature.enabled?(:analytics_devops_adoption_dependency_scanning, enabled_namespace.namespace, default_enabled: :yaml)
+
+        projects_count_with_artifact(Ci::JobArtifact.dependency_list_reports)
+      end
+
+      # rubocop: disable CodeReuse/ActiveRecord
+      def projects_count_with_artifact(artifacts_scope)
+        subquery = artifacts_scope.created_in_time_range(from: range_start, to: range_end)
+          .where(Ci::JobArtifact.arel_table[:project_id].eq(Project.arel_table[:id])).arel.exists
+
+        snapshot_project_ids.each_slice(1000).sum do |project_ids|
+          Project.where(id: project_ids).where(subquery).count
+        end
       end
       # rubocop: enable CodeReuse/ActiveRecord
     end

ee/spec/lib/analytics/devops_adoption/snapshot_calculator_spec.rb

@@ -167,12 +167,14 @@ RSpec.describe Analytics::DevopsAdoption::SnapshotCalculator do
   end
 
   shared_examples 'calculates artifact type count' do |type|
-    it 'returns number of projects with at least 1 sast CI artifact created in given period' do
+    before do
       create(:ee_ci_job_artifact, type, project: project, created_at: 1.year.before(range_end))
       create(:ee_ci_job_artifact, type, project: project, created_at: 1.day.before(range_end))
       create(:ee_ci_job_artifact, type, project: subproject, created_at: 1.week.before(range_end))
       create(:ee_ci_job_artifact, type, created_at: 1.week.before(range_end))
+    end
 
+    it "returns number of projects with at least 1 #{type} CI artifact created in given period" do
       expect(subject).to eq 2
     end
   end
@@ -189,6 +191,12 @@ RSpec.describe Analytics::DevopsAdoption::SnapshotCalculator do
     include_examples 'calculates artifact type count', :dast
   end
 
+  describe 'dependency_scanning_enabled_count' do
+    subject { data[:dependency_scanning_enabled_count] }
+
+    include_examples 'calculates artifact type count', :dependency_scanning
+  end
+
   context 'when snapshot already exists' do
     subject(:data) { described_class.new(enabled_namespace: enabled_namespace, range_end: range_end, snapshot: snapshot).calculate }