Merge branch 'dora-metrics-modeling' into 'master'

Collect Dora Daily Metrics [RUN ALL RSPEC] [RUN AS-IF-FOSS]

See merge request gitlab-org/gitlab!55473

changelogs/unreleased/dora-metrics-modeling.yml

+---
+title: Add DORA daily metrics modeling
+merge_request: 55473
+author:
+type: added

config/feature_categories.yml

@@ -38,6 +38,7 @@
 - design_management
 - devops_reports
 - disaster_recovery
+- dora_metrics
 - dynamic_application_security_testing
 - editor_extension
 - epics

config/feature_flags/development/dora_daily_metrics.yml

+---
+name: dora_daily_metrics
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55473
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/291746
+milestone: '13.10'
+type: development
+group: group::release
+default_enabled: false

config/sidekiq_queues.yml

@@ -100,6 +100,8 @@
   - 1
 - - disallow_two_factor_for_subgroups
   - 1
+- - dora_metrics
+  - 1
 - - elastic_association_indexer
   - 1
 - - elastic_commit_indexer

db/migrate/20210225090801_create_dora_daily_metrics.rb

+# frozen_string_literal: true
+
+class CreateDoraDailyMetrics < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    with_lock_retries do
+      create_table :dora_daily_metrics, if_not_exists: true do |t|
+        t.references :environment, null: false, foreign_key: { on_delete: :cascade }, index: false
+        t.date :date, null: false
+        t.integer :deployment_frequency
+        t.integer :lead_time_for_changes_in_seconds
+
+        t.index [:environment_id, :date], unique: true
+      end
+    end
+
+    add_check_constraint :dora_daily_metrics, "deployment_frequency >= 0", 'dora_daily_metrics_deployment_frequency_positive'
+    add_check_constraint :dora_daily_metrics, "lead_time_for_changes_in_seconds >= 0", 'dora_daily_metrics_lead_time_for_changes_in_seconds_positive'
+  end
+
+  def down
+    with_lock_retries do
+      drop_table :dora_daily_metrics
+    end
+  end
+end

db/migrate/20210303165201_add_index_for_succeeded_deployments.rb

+# frozen_string_literal: true
+
+class AddIndexForSucceededDeployments < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  INDEX_NAME = 'index_deployments_on_environment_id_status_and_finished_at'
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index(:deployments, %i[environment_id status finished_at], name: INDEX_NAME)
+  end
+
+  def down
+    remove_concurrent_index_by_name(:deployments, INDEX_NAME)
+  end
+end

db/schema_migrations/20210225090801

+65f27401a76856d6cb284078204bb1b80797fa344e1a4ef3d9638c296f2d72d7
\ No newline at end of file

db/schema_migrations/20210303165201

+0a5d306735047101692bbdb37aa829bf70a225af6db7213a8c2eb8168f9a30e9
\ No newline at end of file

db/structure.sql

@@ -12025,6 +12025,25 @@ CREATE SEQUENCE diff_note_positions_id_seq
 
 ALTER SEQUENCE diff_note_positions_id_seq OWNED BY diff_note_positions.id;
 
+CREATE TABLE dora_daily_metrics (
+    id bigint NOT NULL,
+    environment_id bigint NOT NULL,
+    date date NOT NULL,
+    deployment_frequency integer,
+    lead_time_for_changes_in_seconds integer,
+    CONSTRAINT dora_daily_metrics_deployment_frequency_positive CHECK ((deployment_frequency >= 0)),
+    CONSTRAINT dora_daily_metrics_lead_time_for_changes_in_seconds_positive CHECK ((lead_time_for_changes_in_seconds >= 0))
+);
+
+CREATE SEQUENCE dora_daily_metrics_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE dora_daily_metrics_id_seq OWNED BY dora_daily_metrics.id;
+
 CREATE TABLE draft_notes (
     id bigint NOT NULL,
     merge_request_id integer NOT NULL,
@@ -19007,6 +19026,8 @@ ALTER TABLE ONLY design_user_mentions ALTER COLUMN id SET DEFAULT nextval('desig
 
 ALTER TABLE ONLY diff_note_positions ALTER COLUMN id SET DEFAULT nextval('diff_note_positions_id_seq'::regclass);
 
+ALTER TABLE ONLY dora_daily_metrics ALTER COLUMN id SET DEFAULT nextval('dora_daily_metrics_id_seq'::regclass);
+
 ALTER TABLE ONLY draft_notes ALTER COLUMN id SET DEFAULT nextval('draft_notes_id_seq'::regclass);
 
 ALTER TABLE ONLY elastic_reindexing_subtasks ALTER COLUMN id SET DEFAULT nextval('elastic_reindexing_subtasks_id_seq'::regclass);
@@ -20215,6 +20236,9 @@ ALTER TABLE ONLY design_user_mentions
 ALTER TABLE ONLY diff_note_positions
     ADD CONSTRAINT diff_note_positions_pkey PRIMARY KEY (id);
 
+ALTER TABLE ONLY dora_daily_metrics
+    ADD CONSTRAINT dora_daily_metrics_pkey PRIMARY KEY (id);
+
 ALTER TABLE ONLY draft_notes
     ADD CONSTRAINT draft_notes_pkey PRIMARY KEY (id);
 
@@ -22095,6 +22119,8 @@ CREATE INDEX index_deployments_on_environment_id_and_id ON deployments USING btr
 
 CREATE INDEX index_deployments_on_environment_id_and_iid_and_project_id ON deployments USING btree (environment_id, iid, project_id);
 
+CREATE INDEX index_deployments_on_environment_id_status_and_finished_at ON deployments USING btree (environment_id, status, finished_at);
+
 CREATE INDEX index_deployments_on_environment_status_sha ON deployments USING btree (environment_id, status, sha);
 
 CREATE INDEX index_deployments_on_id_and_status_and_created_at ON deployments USING btree (id, status, created_at);
@@ -22149,6 +22175,8 @@ CREATE UNIQUE INDEX index_design_user_mentions_on_note_id ON design_user_mention
 
 CREATE UNIQUE INDEX index_diff_note_positions_on_note_id_and_diff_type ON diff_note_positions USING btree (note_id, diff_type);
 
+CREATE UNIQUE INDEX index_dora_daily_metrics_on_environment_id_and_date ON dora_daily_metrics USING btree (environment_id, date);
+
 CREATE INDEX index_draft_notes_on_author_id ON draft_notes USING btree (author_id);
 
 CREATE INDEX index_draft_notes_on_discussion_id ON draft_notes USING btree (discussion_id);
@@ -25137,6 +25165,9 @@ ALTER TABLE ONLY boards_epic_board_positions
 ALTER TABLE ONLY geo_repository_created_events
     ADD CONSTRAINT fk_rails_1f49e46a61 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE;
 
+ALTER TABLE ONLY dora_daily_metrics
+    ADD CONSTRAINT fk_rails_1fd07aff6f FOREIGN KEY (environment_id) REFERENCES environments(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY boards_epic_lists
     ADD CONSTRAINT fk_rails_1fe6b54909 FOREIGN KEY (label_id) REFERENCES labels(id) ON DELETE CASCADE;
 

ee/app/models/dora/daily_metrics.rb

+# frozen_string_literal: true
+
+module Dora
+  # DevOps Research and Assessment (DORA) key metrics. Deployment Frequency,
+  # Lead Time for Changes, Change Failure Rate and Time to Restore Service
+  # are tracked as daily summary.
+  # Reference: https://cloud.google.com/blog/products/devops-sre/using-the-four-keys-to-measure-your-devops-performance
+  class DailyMetrics < ApplicationRecord
+    belongs_to :environment
+
+    self.table_name = 'dora_daily_metrics'
+
+    class << self
+      def refresh!(environment, date)
+        raise ArgumentError unless environment.is_a?(::Environment) && date.is_a?(Date)
+
+        deployment_frequency = deployment_frequency(environment, date)
+        lead_time_for_changes = lead_time_for_changes(environment, date)
+
+        # This query is concurrent safe upsert with the unique index.
+        connection.execute(<<~SQL)
+          INSERT INTO #{table_name} (
+            environment_id,
+            date,
+            deployment_frequency,
+            lead_time_for_changes_in_seconds
+          )
+          VALUES (
+            #{environment.id},
+            #{ActiveRecord::Base.connection.quote(date.to_s)},
+            (#{deployment_frequency}),
+            (#{lead_time_for_changes})
+          )
+          ON CONFLICT (environment_id, date)
+          DO UPDATE SET
+            deployment_frequency = (#{deployment_frequency}),
+            lead_time_for_changes_in_seconds = (#{lead_time_for_changes})
+        SQL
+      end
+
+      private
+
+      # Compose a query to calculate "Deployment Frequency" of the date
+      def deployment_frequency(environment, date)
+        deployments = Deployment.arel_table
+
+        deployments
+          .project(deployments[:id].count)
+          .where(eligible_deployments(environment, date))
+          .to_sql
+      end
+
+      # Compose a query to calculate "Lead Time for Changes" of the date
+      def lead_time_for_changes(environment, date)
+        deployments = Deployment.arel_table
+        deployment_merge_requests = DeploymentMergeRequest.arel_table
+        merge_request_metrics = MergeRequest::Metrics.arel_table
+
+        deployments
+          .project(
+            Arel.sql(
+              'PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY EXTRACT(EPOCH FROM (deployments.finished_at - merge_request_metrics.merged_at)))'
+            )
+          )
+          .join(deployment_merge_requests).on(
+            deployment_merge_requests[:deployment_id].eq(deployments[:id])
+          )
+          .join(merge_request_metrics).on(
+            merge_request_metrics[:merge_request_id].eq(deployment_merge_requests[:merge_request_id])
+          )
+          .where(eligible_deployments(environment, date))
+          .to_sql
+      end
+
+      def eligible_deployments(environment, date)
+        deployments = Deployment.arel_table
+
+        [deployments[:environment_id].eq(environment.id),
+         deployments[:finished_at].gteq(date.beginning_of_day),
+         deployments[:finished_at].lteq(date.end_of_day),
+         deployments[:status].eq(Deployment.statuses[:success])].reduce(&:and)
+      end
+    end
+  end
+end

ee/app/models/ee/deployment.rb

@@ -10,6 +10,22 @@ module EE
 
     prepended do
       include UsageStatistics
+
+      state_machine :status do
+        after_transition any => :success do |deployment|
+          next unless ::Feature.enabled?(:dora_daily_metrics, deployment.project, default_enabled: :yaml)
+
+          deployment.run_after_commit do
+            # Schedule to refresh the DORA daily metrics.
+            # It has 5 minutes delay due to waiting for the other async processes
+            # (e.g. `LinkMergeRequestWorker`) to be finished before collecting metrics.
+            ::Dora::DailyMetrics::RefreshWorker
+              .perform_in(5.minutes,
+                          deployment.environment_id,
+                          Time.current.to_date.to_s)
+          end
+        end
+      end
     end
   end
 end

ee/app/models/ee/environment.rb

@@ -7,6 +7,8 @@ module EE
     include ::Gitlab::Utils::StrongMemoize
 
     prepended do
+      has_many :dora_daily_metrics, class_name: 'Dora::DailyMetrics'
+
       # Returns environments where its latest deployment is to a cluster
       scope :deployed_to_cluster, -> (cluster) do
         environments = model.arel_table

ee/app/workers/all_queues.yml

@@ -339,6 +339,14 @@
   :weight: 3
   :idempotent: true
   :tags: []
+- :name: dora_metrics:dora_daily_metrics_refresh
+  :feature_category: :dora_metrics
+  :has_external_dependencies:
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: true
+  :tags: []
 - :name: epics:epics_update_epics_dates
   :feature_category: :epics
   :has_external_dependencies:

ee/app/workers/dora/daily_metrics/refresh_worker.rb

+# frozen_string_literal: true
+
+module Dora
+  class DailyMetrics
+    class RefreshWorker
+      include ApplicationWorker
+
+      deduplicate :until_executing
+      idempotent!
+      queue_namespace :dora_metrics
+      feature_category :dora_metrics
+
+      def perform(environment_id, date)
+        Environment.find_by_id(environment_id).try do |environment|
+          ::Dora::DailyMetrics.refresh!(environment, Date.parse(date))
+        end
+      end
+    end
+  end
+end

ee/spec/factories/dora/daily_metrics.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :dora_daily_metrics, class: 'Dora::DailyMetrics' do
+    environment
+    date { Time.current.to_date }
+  end
+end

ee/spec/models/deployment_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Deployment do
+  describe 'state machine' do
+    context 'when deployment succeeded' do
+      let(:deployment) { create(:deployment, :running) }
+
+      it 'schedules Dora::DailyMetrics::RefreshWorker' do
+        freeze_time do
+          expect(::Dora::DailyMetrics::RefreshWorker)
+            .to receive(:perform_in).with(
+              5.minutes,
+              deployment.environment_id,
+              Time.current.to_date.to_s)
+
+          deployment.succeed!
+        end
+      end
+
+      context 'when dora_daily_metrics feature flag is disabled' do
+        before do
+          stub_feature_flags(dora_daily_metrics: false)
+        end
+
+        it 'does not schedule Dora::DailyMetrics::RefreshWorker' do
+          expect(::Dora::DailyMetrics::RefreshWorker).not_to receive(:perform_in)
+
+          deployment.succeed!
+        end
+      end
+    end
+  end
+end

ee/spec/models/dora/daily_metrics_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Dora::DailyMetrics, type: :model do
+  describe 'associations' do
+    it { is_expected.to belong_to(:environment) }
+  end
+
+  describe '.refresh!' do
+    subject { described_class.refresh!(environment, date) }
+
+    around do |example|
+      freeze_time { example.run }
+    end
+
+    let_it_be(:project) { create(:project, :repository) }
+    let_it_be(:environment) { create(:environment, project: project) }
+    let(:date) { 1.day.ago.to_date }
+
+    context 'with finished deployments' do
+      before do
+        # Deployment finished before the date
+        previous_date = date - 1.day
+        create(:deployment, :success, environment: environment, finished_at: previous_date)
+        create(:deployment, :failed, environment: environment, finished_at: previous_date)
+
+        # Deployment finished on the date
+        create(:deployment, :success, environment: environment, finished_at: date)
+        create(:deployment, :failed, environment: environment, finished_at: date)
+
+        # Deployment finished after the date
+        next_date = date + 1.day
+        create(:deployment, :success, environment: environment, finished_at: next_date)
+        create(:deployment, :failed, environment: environment, finished_at: next_date)
+      end
+
+      it 'inserts the daily metrics' do
+        expect { subject }.to change { Dora::DailyMetrics.count }.by(1)
+
+        metrics = environment.dora_daily_metrics.find_by_date(date)
+        expect(metrics.deployment_frequency).to eq(1)
+        expect(metrics.lead_time_for_changes_in_seconds).to be_nil
+      end
+
+      context 'when there is an existing daily metric' do
+        before do
+          create(:dora_daily_metrics, environment: environment, date: date, deployment_frequency: 0)
+        end
+
+        it 'updates the daily metrics' do
+          expect { subject }.not_to change { Dora::DailyMetrics.count }
+
+          metrics = environment.dora_daily_metrics.find_by_date(date)
+          expect(metrics.deployment_frequency).to eq(1)
+        end
+      end
+    end
+
+    context 'with finished deployments and merged MRs' do
+      before do
+        merge_requests = []
+
+        # Merged 1 day ago
+        merge_requests << create(:merge_request, :with_merged_metrics, project: project).tap do |merge_request|
+          merge_request.metrics.update!(merged_at: date - 1.day)
+        end
+
+        # Merged 2 days ago
+        merge_requests << create(:merge_request, :with_merged_metrics, project: project).tap do |merge_request|
+          merge_request.metrics.update!(merged_at: date - 2.days)
+        end
+
+        # Merged 3 days ago
+        merge_requests << create(:merge_request, :with_merged_metrics, project: project).tap do |merge_request|
+          merge_request.metrics.update!(merged_at: date - 3.days)
+        end
+
+        # Deployment finished on the date
+        create(:deployment, :success, environment: environment, finished_at: date, merge_requests: merge_requests)
+      end
+
+      it 'inserts the daily metrics' do
+        subject
+
+        metrics = environment.dora_daily_metrics.find_by_date(date)
+        expect(metrics.lead_time_for_changes_in_seconds).to eq(2.days.to_i) # median
+      end
+
+      context 'when there is an existing daily metric' do
+        let!(:dora_daily_metrics) { create(:dora_daily_metrics, environment: environment, date: date, lead_time_for_changes_in_seconds: nil) }
+
+        it 'updates the daily metrics' do
+          expect { subject }
+            .to change { dora_daily_metrics.reload.lead_time_for_changes_in_seconds }
+            .from(nil)
+            .to(2.days.to_i)
+        end
+      end
+    end
+
+    context 'when date is invalid type' do
+      let(:date) { '2021-02-03' }
+
+      it 'raises an error' do
+        expect { subject }.to raise_error(ArgumentError)
+      end
+    end
+  end
+end

ee/spec/models/environment_spec.rb

@@ -8,6 +8,8 @@ RSpec.describe Environment, :use_clean_rails_memory_store_caching do
   let(:project) { create(:project, :repository) }
   let(:environment) { create(:environment, project: project) }
 
+  it { is_expected.to have_many(:dora_daily_metrics) }
+
   describe '.deployed_to_cluster' do
     let!(:environment) { create(:environment) }
 

ee/spec/workers/dora/daily_metrics/refresh_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Dora::DailyMetrics::RefreshWorker do
+  let_it_be(:environment) { create(:environment) }
+  let(:worker) { described_class.new }
+
+  describe '#perform' do
+    subject { worker.perform(environment_id, date.to_s) }
+
+    let(:environment_id) { environment.id }
+    let(:date) { Time.current.to_date }
+
+    it 'refreshes the DORA metrics on the environment and date' do
+      expect(::Dora::DailyMetrics).to receive(:refresh!).with(environment, date)
+
+      subject
+    end
+
+    context 'when the date is not parsable' do
+      let(:date) { 'abc' }
+
+      it 'raises an error' do
+        expect { subject }.to raise_error(Date::Error)
+      end
+    end
+
+    context 'when an environment does not exist' do
+      let(:environment_id) { non_existing_record_id }
+
+      it 'does not refresh' do
+        expect(::Dora::DailyMetrics).not_to receive(:refresh!)
+
+        subject
+      end
+    end
+  end
+end