Commit 95a0c76c authored by Sean Arnold's avatar Sean Arnold

Merge branch '299407-dora4-api-support-for-change-failure-rate-backend-service' into 'master'

Introduce API for Change Failure Rate DORA metric

See merge request gitlab-org/gitlab!83514
parents 807dcfa9 c8305d27
# frozen_string_literal: true
class AddDoraIncidentsCount < Gitlab::Database::Migration[1.0]
def change
add_column :dora_daily_metrics, :incidents_count, :integer
end
end
caf0959c3cefb1a3614867feb2c93115fc2b41890624afa0944dc9cfbfdecd59
\ No newline at end of file
...@@ -14472,6 +14472,7 @@ CREATE TABLE dora_daily_metrics ( ...@@ -14472,6 +14472,7 @@ CREATE TABLE dora_daily_metrics (
deployment_frequency integer, deployment_frequency integer,
lead_time_for_changes_in_seconds integer, lead_time_for_changes_in_seconds integer,
time_to_restore_service_in_seconds integer, time_to_restore_service_in_seconds integer,
incidents_count integer,
CONSTRAINT dora_daily_metrics_deployment_frequency_positive CHECK ((deployment_frequency >= 0)), CONSTRAINT dora_daily_metrics_deployment_frequency_positive CHECK ((deployment_frequency >= 0)),
CONSTRAINT dora_daily_metrics_lead_time_for_changes_in_seconds_positive CHECK ((lead_time_for_changes_in_seconds >= 0)) CONSTRAINT dora_daily_metrics_lead_time_for_changes_in_seconds_positive CHECK ((lead_time_for_changes_in_seconds >= 0))
); );
...@@ -62,9 +62,9 @@ GET /groups/:id/dora/metrics ...@@ -62,9 +62,9 @@ GET /groups/:id/dora/metrics
``` ```
| Attribute | Type | Required | Description | | Attribute | Type | Required | Description |
|-------------- |-------- |----------|----------------------- | |-------------- |-------- |----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](../index.md#namespaced-path-encoding) can be accessed by the authenticated user. | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](../index.md#namespaced-path-encoding) can be accessed by the authenticated user. |
| `metric` | string | yes | The [metric name](../../user/analytics/ci_cd_analytics.md#supported-metrics-in-gitlab). One of `deployment_frequency`, `lead_time_for_changes` or `time_to_restore_service`. | | `metric` | string | yes | The [metric name](../../user/analytics/ci_cd_analytics.md#supported-metrics-in-gitlab). One of `deployment_frequency`, `lead_time_for_changes`, `time_to_restore_service` or `change_failure_rate`. |
| `start_date` | string | no | Date range to start from. ISO 8601 Date format, for example `2021-03-01`. Default is 3 months ago. | | `start_date` | string | no | Date range to start from. ISO 8601 Date format, for example `2021-03-01`. Default is 3 months ago. |
| `end_date` | string | no | Date range to end at. ISO 8601 Date format, for example `2021-03-01`. Default is the current date. | | `end_date` | string | no | Date range to end at. ISO 8601 Date format, for example `2021-03-01`. Default is the current date. |
| `interval` | string | no | The bucketing interval. One of `all`, `monthly` or `daily`. Default is `daily`. | | `interval` | string | no | The bucketing interval. One of `all`, `monthly` or `daily`. Default is `daily`. |
...@@ -101,4 +101,5 @@ parameter: ...@@ -101,4 +101,5 @@ parameter:
| ------------------------ |--------------------------------------------------------------------------------------------------------------------------------------------------------------| | ------------------------ |--------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `deployment_frequency` | The number of successful deployments during the time period. | | `deployment_frequency` | The number of successful deployments during the time period. |
| `lead_time_for_changes` | The median number of seconds between the merge of the merge request (MR) and the deployment of the MR's commits for all MRs deployed during the time period. | | `lead_time_for_changes` | The median number of seconds between the merge of the merge request (MR) and the deployment of the MR's commits for all MRs deployed during the time period. |
| `time_to_restore_service` | The median number of seconds an incident was open during the time period. Available only for production environment | | `time_to_restore_service` | The median number of seconds an incident was open during the time period. Available only for production environment. |
| `change_failure_rate` | The number of incidents divided by the number of deployments during the time period. Available only for production environment. |
...@@ -18128,6 +18128,7 @@ All supported DORA metric types. ...@@ -18128,6 +18128,7 @@ All supported DORA metric types.
| Value | Description | | Value | Description |
| ----- | ----------- | | ----- | ----------- |
| <a id="dorametrictypechange_failure_rate"></a>`CHANGE_FAILURE_RATE` | Change failure rate. |
| <a id="dorametrictypedeployment_frequency"></a>`DEPLOYMENT_FREQUENCY` | Deployment frequency. | | <a id="dorametrictypedeployment_frequency"></a>`DEPLOYMENT_FREQUENCY` | Deployment frequency. |
| <a id="dorametrictypelead_time_for_changes"></a>`LEAD_TIME_FOR_CHANGES` | Lead time for changes. | | <a id="dorametrictypelead_time_for_changes"></a>`LEAD_TIME_FOR_CHANGES` | Lead time for changes. |
| <a id="dorametrictypetime_to_restore_service"></a>`TIME_TO_RESTORE_SERVICE` | Time to restore service. | | <a id="dorametrictypetime_to_restore_service"></a>`TIME_TO_RESTORE_SERVICE` | Time to restore service. |
...@@ -8,5 +8,6 @@ module Types ...@@ -8,5 +8,6 @@ module Types
value 'DEPLOYMENT_FREQUENCY', description: 'Deployment frequency.', value: Dora::DailyMetrics::METRIC_DEPLOYMENT_FREQUENCY value 'DEPLOYMENT_FREQUENCY', description: 'Deployment frequency.', value: Dora::DailyMetrics::METRIC_DEPLOYMENT_FREQUENCY
value 'LEAD_TIME_FOR_CHANGES', description: 'Lead time for changes.', value: Dora::DailyMetrics::METRIC_LEAD_TIME_FOR_CHANGES value 'LEAD_TIME_FOR_CHANGES', description: 'Lead time for changes.', value: Dora::DailyMetrics::METRIC_LEAD_TIME_FOR_CHANGES
value 'TIME_TO_RESTORE_SERVICE', description: 'Time to restore service.', value: Dora::DailyMetrics::METRIC_TIME_TO_RESTORE_SERVICE value 'TIME_TO_RESTORE_SERVICE', description: 'Time to restore service.', value: Dora::DailyMetrics::METRIC_TIME_TO_RESTORE_SERVICE
value 'CHANGE_FAILURE_RATE', description: 'Change failure rate.', value: Dora::DailyMetrics::METRIC_CHANGE_FAILURE_RATE
end end
end end
...@@ -16,7 +16,8 @@ module Dora ...@@ -16,7 +16,8 @@ module Dora
METRIC_DEPLOYMENT_FREQUENCY = 'deployment_frequency' METRIC_DEPLOYMENT_FREQUENCY = 'deployment_frequency'
METRIC_LEAD_TIME_FOR_CHANGES = 'lead_time_for_changes' METRIC_LEAD_TIME_FOR_CHANGES = 'lead_time_for_changes'
METRIC_TIME_TO_RESTORE_SERVICE = 'time_to_restore_service' METRIC_TIME_TO_RESTORE_SERVICE = 'time_to_restore_service'
AVAILABLE_METRICS = [METRIC_DEPLOYMENT_FREQUENCY, METRIC_LEAD_TIME_FOR_CHANGES, METRIC_TIME_TO_RESTORE_SERVICE].freeze METRIC_CHANGE_FAILURE_RATE = 'change_failure_rate'
AVAILABLE_METRICS = [METRIC_DEPLOYMENT_FREQUENCY, METRIC_LEAD_TIME_FOR_CHANGES, METRIC_TIME_TO_RESTORE_SERVICE, METRIC_CHANGE_FAILURE_RATE].freeze
AVAILABLE_INTERVALS = [INTERVAL_ALL, INTERVAL_MONTHLY, INTERVAL_DAILY].freeze AVAILABLE_INTERVALS = [INTERVAL_ALL, INTERVAL_MONTHLY, INTERVAL_DAILY].freeze
scope :for_environments, -> (environments) do scope :for_environments, -> (environments) do
...@@ -34,6 +35,7 @@ module Dora ...@@ -34,6 +35,7 @@ module Dora
deployment_frequency = deployment_frequency(environment, date) deployment_frequency = deployment_frequency(environment, date)
lead_time_for_changes = lead_time_for_changes(environment, date) lead_time_for_changes = lead_time_for_changes(environment, date)
time_to_restore_service = time_to_restore_service(environment, date) time_to_restore_service = time_to_restore_service(environment, date)
incidents_count = incidents_count(environment, date)
# This query is concurrent safe upsert with the unique index. # This query is concurrent safe upsert with the unique index.
connection.execute(<<~SQL) connection.execute(<<~SQL)
...@@ -42,20 +44,23 @@ module Dora ...@@ -42,20 +44,23 @@ module Dora
date, date,
deployment_frequency, deployment_frequency,
lead_time_for_changes_in_seconds, lead_time_for_changes_in_seconds,
time_to_restore_service_in_seconds time_to_restore_service_in_seconds,
incidents_count
) )
VALUES ( VALUES (
#{environment.id}, #{environment.id},
#{connection.quote(date.to_s)}, #{connection.quote(date.to_s)},
(#{deployment_frequency}), (#{deployment_frequency}),
(#{lead_time_for_changes}), (#{lead_time_for_changes}),
(#{time_to_restore_service}) (#{time_to_restore_service}),
(#{incidents_count})
) )
ON CONFLICT (environment_id, date) ON CONFLICT (environment_id, date)
DO UPDATE SET DO UPDATE SET
deployment_frequency = (#{deployment_frequency}), deployment_frequency = (#{deployment_frequency}),
lead_time_for_changes_in_seconds = (#{lead_time_for_changes}), lead_time_for_changes_in_seconds = (#{lead_time_for_changes}),
time_to_restore_service_in_seconds = (#{time_to_restore_service}) time_to_restore_service_in_seconds = (#{time_to_restore_service}),
incidents_count = (#{incidents_count})
SQL SQL
end end
...@@ -92,6 +97,8 @@ module Dora ...@@ -92,6 +97,8 @@ module Dora
when METRIC_TIME_TO_RESTORE_SERVICE when METRIC_TIME_TO_RESTORE_SERVICE
# Median # Median
'(PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY time_to_restore_service_in_seconds)) AS data' '(PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY time_to_restore_service_in_seconds)) AS data'
when METRIC_CHANGE_FAILURE_RATE
'SUM(incidents_count)::float / GREATEST(SUM(deployment_frequency), 1) as data'
else else
raise ArgumentError, 'Unknown metric' raise ArgumentError, 'Unknown metric'
end end
...@@ -151,6 +158,15 @@ module Dora ...@@ -151,6 +158,15 @@ module Dora
.where(project_id: environment.project_id) .where(project_id: environment.project_id)
.to_sql .to_sql
end end
def incidents_count(environment, date)
return Arel.sql('NULL') unless environment.production?
Issue.incident.select(Issue.arel_table[:id].count)
.where(created_at: date.beginning_of_day..date.end_of_day)
.where(project_id: environment.project_id)
.to_sql
end
end end
end end
end end
...@@ -95,6 +95,16 @@ module EE ...@@ -95,6 +95,16 @@ module EE
end end
end end
end end
after_commit on: :create do |issue|
next unless issue.incident?
related_production_env = issue.project.environments.production.first
next unless related_production_env
::Dora::DailyMetrics::RefreshWorker.perform_async(related_production_env.id, issue.created_at.to_date.to_s)
end
end end
class_methods do class_methods do
......
...@@ -7,7 +7,8 @@ RSpec.describe Types::DoraMetricTypeEnum do ...@@ -7,7 +7,8 @@ RSpec.describe Types::DoraMetricTypeEnum do
expect(described_class.values).to match( expect(described_class.values).to match(
'DEPLOYMENT_FREQUENCY' => have_attributes(value: 'deployment_frequency'), 'DEPLOYMENT_FREQUENCY' => have_attributes(value: 'deployment_frequency'),
'LEAD_TIME_FOR_CHANGES' => have_attributes(value: 'lead_time_for_changes'), 'LEAD_TIME_FOR_CHANGES' => have_attributes(value: 'lead_time_for_changes'),
'TIME_TO_RESTORE_SERVICE' => have_attributes(value: 'time_to_restore_service') 'TIME_TO_RESTORE_SERVICE' => have_attributes(value: 'time_to_restore_service'),
'CHANGE_FAILURE_RATE' => have_attributes(value: 'change_failure_rate')
) )
end end
end end
...@@ -58,7 +58,7 @@ RSpec.describe Dora::DailyMetrics, type: :model do ...@@ -58,7 +58,7 @@ RSpec.describe Dora::DailyMetrics, type: :model do
end end
describe '.refresh!' do describe '.refresh!' do
subject { described_class.refresh!(environment, date) } subject { described_class.refresh!(environment, date.to_date) }
around do |example| around do |example|
freeze_time { example.run } freeze_time { example.run }
...@@ -67,7 +67,7 @@ RSpec.describe Dora::DailyMetrics, type: :model do ...@@ -67,7 +67,7 @@ RSpec.describe Dora::DailyMetrics, type: :model do
let_it_be(:project) { create(:project, :repository) } let_it_be(:project) { create(:project, :repository) }
let_it_be(:environment) { create(:environment, project: project) } let_it_be(:environment) { create(:environment, project: project) }
let(:date) { 1.day.ago.to_date } let_it_be(:date) { 1.day.ago }
context 'with finished deployments' do context 'with finished deployments' do
before do before do
...@@ -152,16 +152,16 @@ RSpec.describe Dora::DailyMetrics, type: :model do ...@@ -152,16 +152,16 @@ RSpec.describe Dora::DailyMetrics, type: :model do
context 'with closed issues' do context 'with closed issues' do
before do before do
create(:issue, :incident, :closed, project: project, created_at: date - 7.days, closed_at: date) create(:incident, :closed, project: project, created_at: date - 7.days, closed_at: date)
create(:issue, :incident, :closed, project: project, created_at: date - 5.days, closed_at: date) create(:incident, :closed, project: project, created_at: date - 5.days, closed_at: date)
create(:issue, :incident, :closed, project: project, created_at: date - 3.days, closed_at: date) create(:incident, :closed, project: project, created_at: date - 3.days, closed_at: date)
create(:issue, :incident, :closed, project: project, created_at: date - 1.day, closed_at: date) create(:incident, :closed, project: project, created_at: date - 1.day, closed_at: date)
# Issues which shouldn't be included in calculation # Issues which shouldn't be included in calculation
create(:issue, :closed, project: project, created_at: date - 1.year, closed_at: date) # not an incident create(:issue, :closed, project: project, created_at: date - 1.year, closed_at: date) # not an incident
create(:issue, :incident, project: project, created_at: date - 1.year) # not closed yet create(:incident, project: project, created_at: date - 1.year) # not closed yet
create(:issue, :incident, :closed, created_at: date - 1.year, closed_at: date) # different project create(:incident, :closed, created_at: date - 1.year, closed_at: date) # different project
create(:issue, :incident, :closed, project: project, created_at: date - 1.year, closed_at: date + 1.day) # different date create(:incident, :closed, project: project, created_at: date - 1.year, closed_at: date + 1.day) # different date
end end
context 'for production environment' do context 'for production environment' do
...@@ -185,11 +185,45 @@ RSpec.describe Dora::DailyMetrics, type: :model do ...@@ -185,11 +185,45 @@ RSpec.describe Dora::DailyMetrics, type: :model do
end end
end end
context 'when date is invalid type' do context 'with incidents' do
let(:date) { '2021-02-03' } before_all do
create(:incident, project: project, created_at: date.beginning_of_day)
create(:incident, project: project, created_at: date.beginning_of_day + 1.hour)
create(:incident, project: project, created_at: date.end_of_day)
# Issues which shouldn't be included in calculation
create(:issue, project: project, created_at: date) # not an incident
create(:incident, created_at: date) # different project
create(:incident, project: project, created_at: date - 1.year) # different date
create(:incident, project: project, created_at: date + 1.year) # different date
end
context 'for production environment' do
let_it_be(:environment) { create(:environment, :production, project: project) }
it 'inserts the daily metrics with incidents_count' do
subject
metrics = environment.dora_daily_metrics.find_by_date(date)
expect(metrics.incidents_count).to eq(3)
end
end
context 'for non-production environment' do
it 'does not calculate incidents_count daily metric' do
subject
metrics = environment.dora_daily_metrics.find_by_date(date)
expect(metrics.incidents_count).to be_nil
end
end
end
context 'when date is invalid type' do
it 'raises an error' do it 'raises an error' do
expect { subject }.to raise_error(ArgumentError) expect do
described_class.refresh!(environment, '2021-02-03')
end.to raise_error(ArgumentError)
end end
end end
end end
...@@ -250,6 +284,56 @@ RSpec.describe Dora::DailyMetrics, type: :model do ...@@ -250,6 +284,56 @@ RSpec.describe Dora::DailyMetrics, type: :model do
end end
end end
context 'when metric is change_failure_rate' do
let_it_be(:environment) { create :environment }
before_all do
create(:dora_daily_metrics, environment: environment, deployment_frequency: 3, incidents_count: 1, date: '2021-01-01')
create(:dora_daily_metrics, environment: environment, deployment_frequency: 2, incidents_count: 0, date: '2021-01-02')
create(:dora_daily_metrics, environment: environment, deployment_frequency: 2, incidents_count: nil, date: '2021-01-03')
create(:dora_daily_metrics, environment: environment, deployment_frequency: 0, incidents_count: 2, date: '2021-01-04')
create(:dora_daily_metrics, environment: environment, deployment_frequency: nil, incidents_count: nil, date: '2021-01-05')
create(:dora_daily_metrics, environment: environment, deployment_frequency: 0, incidents_count: 0, date: '2021-01-06')
end
let(:metric) { described_class::METRIC_CHANGE_FAILURE_RATE }
context 'when interval is all' do
let(:interval) { described_class::INTERVAL_ALL }
it 'aggregates the rows' do
is_expected.to eq(3 / 7.0)
end
end
context 'when interval is monthly' do
let(:interval) { described_class::INTERVAL_MONTHLY }
it 'aggregates the rows' do
is_expected.to eq([{ 'date' => '2021-01-01', 'value' => 3 / 7.0 }])
end
end
context 'when interval is daily' do
let(:interval) { described_class::INTERVAL_DAILY }
it 'aggregates the rows' do
is_expected.to eq([{ 'date' => '2021-01-01', 'value' => 1 / 3.0 },
{ 'date' => '2021-01-02', 'value' => 0.0 },
{ 'date' => '2021-01-03', 'value' => nil },
{ 'date' => '2021-01-04', 'value' => 2.0 },
{ 'date' => '2021-01-05', 'value' => nil },
{ 'date' => '2021-01-06', 'value' => 0.0 }])
end
end
context 'when interval is unknown' do
let(:interval) { 'unknown' }
it { expect { subject }.to raise_error(ArgumentError, 'Unknown interval') }
end
end
shared_examples 'median metric' do |metric| shared_examples 'median metric' do |metric|
subject { described_class.aggregate_for!(metric, interval) } subject { described_class.aggregate_for!(metric, interval) }
......
...@@ -353,12 +353,12 @@ RSpec.describe Issue do ...@@ -353,12 +353,12 @@ RSpec.describe Issue do
it { is_expected.to have_one(:status_page_published_incident) } it { is_expected.to have_one(:status_page_published_incident) }
end end
describe 'state machine' do describe 'daily dora metrics refresh' do
context 'daily dora metrics refresh' do
let_it_be(:production_env) { create(:environment, :production) } let_it_be(:production_env) { create(:environment, :production) }
context 'on issue close' do
context 'when incident is closed' do context 'when incident is closed' do
let(:issue) { create(:issue, :incident, project: production_env.project) } let!(:issue) { create(:incident, project: production_env.project) }
it 'schedules Dora::DailyMetrics::RefreshWorker' do it 'schedules Dora::DailyMetrics::RefreshWorker' do
freeze_time do freeze_time do
...@@ -371,7 +371,7 @@ RSpec.describe Issue do ...@@ -371,7 +371,7 @@ RSpec.describe Issue do
end end
context 'when there is no production env' do context 'when there is no production env' do
let(:issue) { create(:issue, :incident) } let!(:issue) { create(:incident) }
it 'does not schedule Dora::DailyMetrics::RefreshWorker' do it 'does not schedule Dora::DailyMetrics::RefreshWorker' do
expect(::Dora::DailyMetrics::RefreshWorker).not_to receive(:perform_async) expect(::Dora::DailyMetrics::RefreshWorker).not_to receive(:perform_async)
...@@ -381,7 +381,7 @@ RSpec.describe Issue do ...@@ -381,7 +381,7 @@ RSpec.describe Issue do
end end
context 'when issue is not an incident' do context 'when issue is not an incident' do
let(:issue) { create(:issue, project: production_env.project) } let!(:issue) { create(:issue, project: production_env.project) }
it 'does not schedule Dora::DailyMetrics::RefreshWorker' do it 'does not schedule Dora::DailyMetrics::RefreshWorker' do
expect(::Dora::DailyMetrics::RefreshWorker).not_to receive(:perform_async) expect(::Dora::DailyMetrics::RefreshWorker).not_to receive(:perform_async)
...@@ -390,6 +390,33 @@ RSpec.describe Issue do ...@@ -390,6 +390,33 @@ RSpec.describe Issue do
end end
end end
end end
context 'on incident create' do
it 'schedules Dora::DailyMetrics::RefreshWorker' do
freeze_time do
expect(::Dora::DailyMetrics::RefreshWorker)
.to receive(:perform_async).with(production_env.id, Time.current.to_date.to_s)
create(:incident, project: production_env.project)
end
end
context 'when there is no production env' do
it 'does not schedule Dora::DailyMetrics::RefreshWorker' do
expect(::Dora::DailyMetrics::RefreshWorker).not_to receive(:perform_async)
create(:incident)
end
end
context 'when issue is not an incident' do
it 'does not schedule Dora::DailyMetrics::RefreshWorker' do
expect(::Dora::DailyMetrics::RefreshWorker).not_to receive(:perform_async)
create(:issue, project: production_env.project)
end
end
end
end end
it_behaves_like 'an editable mentionable with EE-specific mentions' do it_behaves_like 'an editable mentionable with EE-specific mentions' do
......
...@@ -25,12 +25,14 @@ RSpec.describe API::Dora::Metrics do ...@@ -25,12 +25,14 @@ RSpec.describe API::Dora::Metrics do
deployment_frequency: 1, deployment_frequency: 1,
lead_time_for_changes_in_seconds: 3, lead_time_for_changes_in_seconds: 3,
time_to_restore_service_in_seconds: 5, time_to_restore_service_in_seconds: 5,
incidents_count: 7,
environment: production, environment: production,
date: '2021-01-01') date: '2021-01-01')
create(:dora_daily_metrics, create(:dora_daily_metrics,
deployment_frequency: 2, deployment_frequency: 2,
lead_time_for_changes_in_seconds: 4, lead_time_for_changes_in_seconds: 4,
time_to_restore_service_in_seconds: 6, time_to_restore_service_in_seconds: 6,
incidents_count: 8,
environment: production, environment: production,
date: '2021-01-02') date: '2021-01-02')
end end
...@@ -43,6 +45,7 @@ RSpec.describe API::Dora::Metrics do ...@@ -43,6 +45,7 @@ RSpec.describe API::Dora::Metrics do
:deployment_frequency | 1 | 2 :deployment_frequency | 1 | 2
:lead_time_for_changes | 3 | 4 :lead_time_for_changes | 3 | 4
:time_to_restore_service | 5 | 6 :time_to_restore_service | 5 | 6
:change_failure_rate | 7 | 4
end end
with_them do with_them do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment