Commit e8a9a795 authored by Dmytro Zaporozhets (DZ)'s avatar Dmytro Zaporozhets (DZ)

Merge branch 'dora-metrics-modeling' into 'master'

Collect Dora Daily Metrics [RUN ALL RSPEC] [RUN AS-IF-FOSS]

See merge request gitlab-org/gitlab!55473
parents 1ece9271 59a00544
---
title: Add DORA daily metrics modeling
merge_request: 55473
author:
type: added
...@@ -38,6 +38,7 @@ ...@@ -38,6 +38,7 @@
- design_management - design_management
- devops_reports - devops_reports
- disaster_recovery - disaster_recovery
- dora_metrics
- dynamic_application_security_testing - dynamic_application_security_testing
- editor_extension - editor_extension
- epics - epics
......
---
name: dora_daily_metrics
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55473
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/291746
milestone: '13.10'
type: development
group: group::release
default_enabled: false
...@@ -100,6 +100,8 @@ ...@@ -100,6 +100,8 @@
- 1 - 1
- - disallow_two_factor_for_subgroups - - disallow_two_factor_for_subgroups
- 1 - 1
- - dora_metrics
- 1
- - elastic_association_indexer - - elastic_association_indexer
- 1 - 1
- - elastic_commit_indexer - - elastic_commit_indexer
......
# frozen_string_literal: true
class CreateDoraDailyMetrics < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
with_lock_retries do
create_table :dora_daily_metrics, if_not_exists: true do |t|
t.references :environment, null: false, foreign_key: { on_delete: :cascade }, index: false
t.date :date, null: false
t.integer :deployment_frequency
t.integer :lead_time_for_changes_in_seconds
t.index [:environment_id, :date], unique: true
end
end
add_check_constraint :dora_daily_metrics, "deployment_frequency >= 0", 'dora_daily_metrics_deployment_frequency_positive'
add_check_constraint :dora_daily_metrics, "lead_time_for_changes_in_seconds >= 0", 'dora_daily_metrics_lead_time_for_changes_in_seconds_positive'
end
def down
with_lock_retries do
drop_table :dora_daily_metrics
end
end
end
# frozen_string_literal: true
class AddIndexForSucceededDeployments < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
INDEX_NAME = 'index_deployments_on_environment_id_status_and_finished_at'
disable_ddl_transaction!
def up
add_concurrent_index(:deployments, %i[environment_id status finished_at], name: INDEX_NAME)
end
def down
remove_concurrent_index_by_name(:deployments, INDEX_NAME)
end
end
65f27401a76856d6cb284078204bb1b80797fa344e1a4ef3d9638c296f2d72d7
\ No newline at end of file
0a5d306735047101692bbdb37aa829bf70a225af6db7213a8c2eb8168f9a30e9
\ No newline at end of file
...@@ -12025,6 +12025,25 @@ CREATE SEQUENCE diff_note_positions_id_seq ...@@ -12025,6 +12025,25 @@ CREATE SEQUENCE diff_note_positions_id_seq
ALTER SEQUENCE diff_note_positions_id_seq OWNED BY diff_note_positions.id; ALTER SEQUENCE diff_note_positions_id_seq OWNED BY diff_note_positions.id;
CREATE TABLE dora_daily_metrics (
id bigint NOT NULL,
environment_id bigint NOT NULL,
date date NOT NULL,
deployment_frequency integer,
lead_time_for_changes_in_seconds integer,
CONSTRAINT dora_daily_metrics_deployment_frequency_positive CHECK ((deployment_frequency >= 0)),
CONSTRAINT dora_daily_metrics_lead_time_for_changes_in_seconds_positive CHECK ((lead_time_for_changes_in_seconds >= 0))
);
CREATE SEQUENCE dora_daily_metrics_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE dora_daily_metrics_id_seq OWNED BY dora_daily_metrics.id;
CREATE TABLE draft_notes ( CREATE TABLE draft_notes (
id bigint NOT NULL, id bigint NOT NULL,
merge_request_id integer NOT NULL, merge_request_id integer NOT NULL,
...@@ -19007,6 +19026,8 @@ ALTER TABLE ONLY design_user_mentions ALTER COLUMN id SET DEFAULT nextval('desig ...@@ -19007,6 +19026,8 @@ ALTER TABLE ONLY design_user_mentions ALTER COLUMN id SET DEFAULT nextval('desig
ALTER TABLE ONLY diff_note_positions ALTER COLUMN id SET DEFAULT nextval('diff_note_positions_id_seq'::regclass); ALTER TABLE ONLY diff_note_positions ALTER COLUMN id SET DEFAULT nextval('diff_note_positions_id_seq'::regclass);
ALTER TABLE ONLY dora_daily_metrics ALTER COLUMN id SET DEFAULT nextval('dora_daily_metrics_id_seq'::regclass);
ALTER TABLE ONLY draft_notes ALTER COLUMN id SET DEFAULT nextval('draft_notes_id_seq'::regclass); ALTER TABLE ONLY draft_notes ALTER COLUMN id SET DEFAULT nextval('draft_notes_id_seq'::regclass);
ALTER TABLE ONLY elastic_reindexing_subtasks ALTER COLUMN id SET DEFAULT nextval('elastic_reindexing_subtasks_id_seq'::regclass); ALTER TABLE ONLY elastic_reindexing_subtasks ALTER COLUMN id SET DEFAULT nextval('elastic_reindexing_subtasks_id_seq'::regclass);
...@@ -20215,6 +20236,9 @@ ALTER TABLE ONLY design_user_mentions ...@@ -20215,6 +20236,9 @@ ALTER TABLE ONLY design_user_mentions
ALTER TABLE ONLY diff_note_positions ALTER TABLE ONLY diff_note_positions
ADD CONSTRAINT diff_note_positions_pkey PRIMARY KEY (id); ADD CONSTRAINT diff_note_positions_pkey PRIMARY KEY (id);
ALTER TABLE ONLY dora_daily_metrics
ADD CONSTRAINT dora_daily_metrics_pkey PRIMARY KEY (id);
ALTER TABLE ONLY draft_notes ALTER TABLE ONLY draft_notes
ADD CONSTRAINT draft_notes_pkey PRIMARY KEY (id); ADD CONSTRAINT draft_notes_pkey PRIMARY KEY (id);
...@@ -22095,6 +22119,8 @@ CREATE INDEX index_deployments_on_environment_id_and_id ON deployments USING btr ...@@ -22095,6 +22119,8 @@ CREATE INDEX index_deployments_on_environment_id_and_id ON deployments USING btr
CREATE INDEX index_deployments_on_environment_id_and_iid_and_project_id ON deployments USING btree (environment_id, iid, project_id); CREATE INDEX index_deployments_on_environment_id_and_iid_and_project_id ON deployments USING btree (environment_id, iid, project_id);
CREATE INDEX index_deployments_on_environment_id_status_and_finished_at ON deployments USING btree (environment_id, status, finished_at);
CREATE INDEX index_deployments_on_environment_status_sha ON deployments USING btree (environment_id, status, sha); CREATE INDEX index_deployments_on_environment_status_sha ON deployments USING btree (environment_id, status, sha);
CREATE INDEX index_deployments_on_id_and_status_and_created_at ON deployments USING btree (id, status, created_at); CREATE INDEX index_deployments_on_id_and_status_and_created_at ON deployments USING btree (id, status, created_at);
...@@ -22149,6 +22175,8 @@ CREATE UNIQUE INDEX index_design_user_mentions_on_note_id ON design_user_mention ...@@ -22149,6 +22175,8 @@ CREATE UNIQUE INDEX index_design_user_mentions_on_note_id ON design_user_mention
CREATE UNIQUE INDEX index_diff_note_positions_on_note_id_and_diff_type ON diff_note_positions USING btree (note_id, diff_type); CREATE UNIQUE INDEX index_diff_note_positions_on_note_id_and_diff_type ON diff_note_positions USING btree (note_id, diff_type);
CREATE UNIQUE INDEX index_dora_daily_metrics_on_environment_id_and_date ON dora_daily_metrics USING btree (environment_id, date);
CREATE INDEX index_draft_notes_on_author_id ON draft_notes USING btree (author_id); CREATE INDEX index_draft_notes_on_author_id ON draft_notes USING btree (author_id);
CREATE INDEX index_draft_notes_on_discussion_id ON draft_notes USING btree (discussion_id); CREATE INDEX index_draft_notes_on_discussion_id ON draft_notes USING btree (discussion_id);
...@@ -25137,6 +25165,9 @@ ALTER TABLE ONLY boards_epic_board_positions ...@@ -25137,6 +25165,9 @@ ALTER TABLE ONLY boards_epic_board_positions
ALTER TABLE ONLY geo_repository_created_events ALTER TABLE ONLY geo_repository_created_events
ADD CONSTRAINT fk_rails_1f49e46a61 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; ADD CONSTRAINT fk_rails_1f49e46a61 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE;
ALTER TABLE ONLY dora_daily_metrics
ADD CONSTRAINT fk_rails_1fd07aff6f FOREIGN KEY (environment_id) REFERENCES environments(id) ON DELETE CASCADE;
ALTER TABLE ONLY boards_epic_lists ALTER TABLE ONLY boards_epic_lists
ADD CONSTRAINT fk_rails_1fe6b54909 FOREIGN KEY (label_id) REFERENCES labels(id) ON DELETE CASCADE; ADD CONSTRAINT fk_rails_1fe6b54909 FOREIGN KEY (label_id) REFERENCES labels(id) ON DELETE CASCADE;
# frozen_string_literal: true
module Dora
# DevOps Research and Assessment (DORA) key metrics. Deployment Frequency,
# Lead Time for Changes, Change Failure Rate and Time to Restore Service
# are tracked as daily summary.
# Reference: https://cloud.google.com/blog/products/devops-sre/using-the-four-keys-to-measure-your-devops-performance
class DailyMetrics < ApplicationRecord
belongs_to :environment
self.table_name = 'dora_daily_metrics'
class << self
def refresh!(environment, date)
raise ArgumentError unless environment.is_a?(::Environment) && date.is_a?(Date)
deployment_frequency = deployment_frequency(environment, date)
lead_time_for_changes = lead_time_for_changes(environment, date)
# This query is concurrent safe upsert with the unique index.
connection.execute(<<~SQL)
INSERT INTO #{table_name} (
environment_id,
date,
deployment_frequency,
lead_time_for_changes_in_seconds
)
VALUES (
#{environment.id},
#{ActiveRecord::Base.connection.quote(date.to_s)},
(#{deployment_frequency}),
(#{lead_time_for_changes})
)
ON CONFLICT (environment_id, date)
DO UPDATE SET
deployment_frequency = (#{deployment_frequency}),
lead_time_for_changes_in_seconds = (#{lead_time_for_changes})
SQL
end
private
# Compose a query to calculate "Deployment Frequency" of the date
def deployment_frequency(environment, date)
deployments = Deployment.arel_table
deployments
.project(deployments[:id].count)
.where(eligible_deployments(environment, date))
.to_sql
end
# Compose a query to calculate "Lead Time for Changes" of the date
def lead_time_for_changes(environment, date)
deployments = Deployment.arel_table
deployment_merge_requests = DeploymentMergeRequest.arel_table
merge_request_metrics = MergeRequest::Metrics.arel_table
deployments
.project(
Arel.sql(
'PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY EXTRACT(EPOCH FROM (deployments.finished_at - merge_request_metrics.merged_at)))'
)
)
.join(deployment_merge_requests).on(
deployment_merge_requests[:deployment_id].eq(deployments[:id])
)
.join(merge_request_metrics).on(
merge_request_metrics[:merge_request_id].eq(deployment_merge_requests[:merge_request_id])
)
.where(eligible_deployments(environment, date))
.to_sql
end
def eligible_deployments(environment, date)
deployments = Deployment.arel_table
[deployments[:environment_id].eq(environment.id),
deployments[:finished_at].gteq(date.beginning_of_day),
deployments[:finished_at].lteq(date.end_of_day),
deployments[:status].eq(Deployment.statuses[:success])].reduce(&:and)
end
end
end
end
...@@ -10,6 +10,22 @@ module EE ...@@ -10,6 +10,22 @@ module EE
prepended do prepended do
include UsageStatistics include UsageStatistics
state_machine :status do
after_transition any => :success do |deployment|
next unless ::Feature.enabled?(:dora_daily_metrics, deployment.project, default_enabled: :yaml)
deployment.run_after_commit do
# Schedule to refresh the DORA daily metrics.
# It has 5 minutes delay due to waiting for the other async processes
# (e.g. `LinkMergeRequestWorker`) to be finished before collecting metrics.
::Dora::DailyMetrics::RefreshWorker
.perform_in(5.minutes,
deployment.environment_id,
Time.current.to_date.to_s)
end
end
end
end end
end end
end end
...@@ -7,6 +7,8 @@ module EE ...@@ -7,6 +7,8 @@ module EE
include ::Gitlab::Utils::StrongMemoize include ::Gitlab::Utils::StrongMemoize
prepended do prepended do
has_many :dora_daily_metrics, class_name: 'Dora::DailyMetrics'
# Returns environments where its latest deployment is to a cluster # Returns environments where its latest deployment is to a cluster
scope :deployed_to_cluster, -> (cluster) do scope :deployed_to_cluster, -> (cluster) do
environments = model.arel_table environments = model.arel_table
......
...@@ -339,6 +339,14 @@ ...@@ -339,6 +339,14 @@
:weight: 3 :weight: 3
:idempotent: true :idempotent: true
:tags: [] :tags: []
- :name: dora_metrics:dora_daily_metrics_refresh
:feature_category: :dora_metrics
:has_external_dependencies:
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: true
:tags: []
- :name: epics:epics_update_epics_dates - :name: epics:epics_update_epics_dates
:feature_category: :epics :feature_category: :epics
:has_external_dependencies: :has_external_dependencies:
......
# frozen_string_literal: true
module Dora
class DailyMetrics
class RefreshWorker
include ApplicationWorker
deduplicate :until_executing
idempotent!
queue_namespace :dora_metrics
feature_category :dora_metrics
def perform(environment_id, date)
Environment.find_by_id(environment_id).try do |environment|
::Dora::DailyMetrics.refresh!(environment, Date.parse(date))
end
end
end
end
end
# frozen_string_literal: true
FactoryBot.define do
factory :dora_daily_metrics, class: 'Dora::DailyMetrics' do
environment
date { Time.current.to_date }
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Deployment do
describe 'state machine' do
context 'when deployment succeeded' do
let(:deployment) { create(:deployment, :running) }
it 'schedules Dora::DailyMetrics::RefreshWorker' do
freeze_time do
expect(::Dora::DailyMetrics::RefreshWorker)
.to receive(:perform_in).with(
5.minutes,
deployment.environment_id,
Time.current.to_date.to_s)
deployment.succeed!
end
end
context 'when dora_daily_metrics feature flag is disabled' do
before do
stub_feature_flags(dora_daily_metrics: false)
end
it 'does not schedule Dora::DailyMetrics::RefreshWorker' do
expect(::Dora::DailyMetrics::RefreshWorker).not_to receive(:perform_in)
deployment.succeed!
end
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Dora::DailyMetrics, type: :model do
describe 'associations' do
it { is_expected.to belong_to(:environment) }
end
describe '.refresh!' do
subject { described_class.refresh!(environment, date) }
around do |example|
freeze_time { example.run }
end
let_it_be(:project) { create(:project, :repository) }
let_it_be(:environment) { create(:environment, project: project) }
let(:date) { 1.day.ago.to_date }
context 'with finished deployments' do
before do
# Deployment finished before the date
previous_date = date - 1.day
create(:deployment, :success, environment: environment, finished_at: previous_date)
create(:deployment, :failed, environment: environment, finished_at: previous_date)
# Deployment finished on the date
create(:deployment, :success, environment: environment, finished_at: date)
create(:deployment, :failed, environment: environment, finished_at: date)
# Deployment finished after the date
next_date = date + 1.day
create(:deployment, :success, environment: environment, finished_at: next_date)
create(:deployment, :failed, environment: environment, finished_at: next_date)
end
it 'inserts the daily metrics' do
expect { subject }.to change { Dora::DailyMetrics.count }.by(1)
metrics = environment.dora_daily_metrics.find_by_date(date)
expect(metrics.deployment_frequency).to eq(1)
expect(metrics.lead_time_for_changes_in_seconds).to be_nil
end
context 'when there is an existing daily metric' do
before do
create(:dora_daily_metrics, environment: environment, date: date, deployment_frequency: 0)
end
it 'updates the daily metrics' do
expect { subject }.not_to change { Dora::DailyMetrics.count }
metrics = environment.dora_daily_metrics.find_by_date(date)
expect(metrics.deployment_frequency).to eq(1)
end
end
end
context 'with finished deployments and merged MRs' do
before do
merge_requests = []
# Merged 1 day ago
merge_requests << create(:merge_request, :with_merged_metrics, project: project).tap do |merge_request|
merge_request.metrics.update!(merged_at: date - 1.day)
end
# Merged 2 days ago
merge_requests << create(:merge_request, :with_merged_metrics, project: project).tap do |merge_request|
merge_request.metrics.update!(merged_at: date - 2.days)
end
# Merged 3 days ago
merge_requests << create(:merge_request, :with_merged_metrics, project: project).tap do |merge_request|
merge_request.metrics.update!(merged_at: date - 3.days)
end
# Deployment finished on the date
create(:deployment, :success, environment: environment, finished_at: date, merge_requests: merge_requests)
end
it 'inserts the daily metrics' do
subject
metrics = environment.dora_daily_metrics.find_by_date(date)
expect(metrics.lead_time_for_changes_in_seconds).to eq(2.days.to_i) # median
end
context 'when there is an existing daily metric' do
let!(:dora_daily_metrics) { create(:dora_daily_metrics, environment: environment, date: date, lead_time_for_changes_in_seconds: nil) }
it 'updates the daily metrics' do
expect { subject }
.to change { dora_daily_metrics.reload.lead_time_for_changes_in_seconds }
.from(nil)
.to(2.days.to_i)
end
end
end
context 'when date is invalid type' do
let(:date) { '2021-02-03' }
it 'raises an error' do
expect { subject }.to raise_error(ArgumentError)
end
end
end
end
...@@ -8,6 +8,8 @@ RSpec.describe Environment, :use_clean_rails_memory_store_caching do ...@@ -8,6 +8,8 @@ RSpec.describe Environment, :use_clean_rails_memory_store_caching do
let(:project) { create(:project, :repository) } let(:project) { create(:project, :repository) }
let(:environment) { create(:environment, project: project) } let(:environment) { create(:environment, project: project) }
it { is_expected.to have_many(:dora_daily_metrics) }
describe '.deployed_to_cluster' do describe '.deployed_to_cluster' do
let!(:environment) { create(:environment) } let!(:environment) { create(:environment) }
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Dora::DailyMetrics::RefreshWorker do
let_it_be(:environment) { create(:environment) }
let(:worker) { described_class.new }
describe '#perform' do
subject { worker.perform(environment_id, date.to_s) }
let(:environment_id) { environment.id }
let(:date) { Time.current.to_date }
it 'refreshes the DORA metrics on the environment and date' do
expect(::Dora::DailyMetrics).to receive(:refresh!).with(environment, date)
subject
end
context 'when the date is not parsable' do
let(:date) { 'abc' }
it 'raises an error' do
expect { subject }.to raise_error(Date::Error)
end
end
context 'when an environment does not exist' do
let(:environment_id) { non_existing_record_id }
it 'does not refresh' do
expect(::Dora::DailyMetrics).not_to receive(:refresh!)
subject
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment