Commit 2d7cbc8a authored by Sean McGivern's avatar Sean McGivern

Merge branch 'milestone-burnup-charts-backend' into 'master'

Create a service to compute burnup charts for milestones

See merge request gitlab-org/gitlab!37636
parents c2ad5dbc 5ed9595e
---
title: Add index to resource_milestone_events for add actions
merge_request: 37636
author:
type: other
# frozen_string_literal: true
class AddIndexToResourceMilestoneEventsAddEvents < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
INDEX_NAME = 'index_resource_milestone_events_on_milestone_id_and_add_action'
ADD_ACTION = '1'
def up
# Index add milestone events
add_concurrent_index :resource_milestone_events, :milestone_id, where: "action = #{ADD_ACTION}", name: INDEX_NAME
end
def down
remove_concurrent_index :resource_milestone_events, :milestone_id, name: INDEX_NAME
end
end
5dc4cbfc6d7e79e5909e5250f382bc3c9fa4246b8f2aed81404899aee4eef81b
\ No newline at end of file
......@@ -20468,6 +20468,8 @@ CREATE INDEX index_resource_milestone_events_on_merge_request_id ON public.resou
CREATE INDEX index_resource_milestone_events_on_milestone_id ON public.resource_milestone_events USING btree (milestone_id);
CREATE INDEX index_resource_milestone_events_on_milestone_id_and_add_action ON public.resource_milestone_events USING btree (milestone_id) WHERE (action = 1);
CREATE INDEX index_resource_milestone_events_on_user_id ON public.resource_milestone_events USING btree (user_id);
CREATE INDEX index_resource_state_events_on_epic_id ON public.resource_state_events USING btree (epic_id);
......
# frozen_string_literal: true
# This service computes the milestone's daily total number of issues and their weights.
# For each day, this returns the totals for all issues that are assigned to the milestone at that point in time.
# This represents the scope for this milestone. This also returns separate totals for closed issues which represent the completed work.
#
# This is implemented by iterating over all relevant resource events ordered by time. We need to do this
# so that we can keep track of the issue's state during that point in time and handle the events based on that.
class Milestones::BurnupChartService
include Gitlab::Utils::StrongMemoize
EVENT_COUNT_LIMIT = 50_000
TooManyEventsError = Class.new(StandardError)
def initialize(milestone)
raise ArgumentError, 'Milestone must have a start and due date' if milestone.start_date.blank? || milestone.due_date.blank?
@milestone = milestone
end
def execute
@issue_states = {}
@chart_data = []
raise TooManyEventsError if resource_events.num_tuples > EVENT_COUNT_LIMIT
resource_events.each do |event|
case event['event_type']
when 'milestone'
handle_milestone_event(event)
when 'state'
handle_state_event(event)
when 'weight'
handle_weight_event(event)
end
end
chart_data
end
private
attr_reader :milestone, :issue_states, :chart_data
def handle_milestone_event(event)
issue_state = find_issue_state(event['issue_id'])
return if issue_state[:milestone] == milestone.id && event['action'] == ResourceMilestoneEvent.actions[:add] && event['value'] == milestone.id
if event['action'] == ResourceMilestoneEvent.actions[:add] && event['value'] == milestone.id
handle_add_milestone_event(event)
elsif issue_state[:milestone] == milestone.id
# If the issue is currently assigned to the milestone, then treat any event here as a removal.
# We do not have a separate `:remove` event when replacing milestones with another one.
handle_remove_milestone_event(event)
end
issue_state[:milestone] = event['value']
end
def handle_add_milestone_event(event)
issue_state = find_issue_state(event['issue_id'])
increment_scope(event['created_at'], issue_state[:weight])
if issue_state[:state] == ResourceStateEvent.states[:closed]
increment_completed(event['created_at'], issue_state[:weight])
end
end
def handle_remove_milestone_event(event)
issue_state = find_issue_state(event['issue_id'])
decrement_scope(event['created_at'], issue_state[:weight])
if issue_state[:state] == ResourceStateEvent.states[:closed]
decrement_completed(event['created_at'], issue_state[:weight])
end
end
def handle_state_event(event)
issue_state = find_issue_state(event['issue_id'])
old_state = issue_state[:state]
issue_state[:state] = event['value']
return if issue_state[:milestone] != milestone.id
if old_state == ResourceStateEvent.states[:closed] && event['value'] == ResourceStateEvent.states[:reopened]
decrement_completed(event['created_at'], issue_state[:weight])
elsif ResourceStateEvent.states.values_at(:opened, :reopened).include?(old_state) && event['value'] == ResourceStateEvent.states[:closed]
increment_completed(event['created_at'], issue_state[:weight])
end
end
def handle_weight_event(event)
issue_state = find_issue_state(event['issue_id'])
old_weight = issue_state[:weight]
issue_state[:weight] = event['value'] || 0
return if issue_state[:milestone] != milestone.id
add_chart_data(event['created_at'], :scope_weight, issue_state[:weight] - old_weight)
if issue_state[:state] == ResourceStateEvent.states[:closed]
add_chart_data(event['created_at'], :completed_weight, issue_state[:weight] - old_weight)
end
end
def increment_scope(timestamp, weight)
add_chart_data(timestamp, :scope_count, 1)
add_chart_data(timestamp, :scope_weight, weight)
end
def decrement_scope(timestamp, weight)
add_chart_data(timestamp, :scope_count, -1)
add_chart_data(timestamp, :scope_weight, -weight)
end
def increment_completed(timestamp, weight)
add_chart_data(timestamp, :completed_count, 1)
add_chart_data(timestamp, :completed_weight, weight)
end
def decrement_completed(timestamp, weight)
add_chart_data(timestamp, :completed_count, -1)
add_chart_data(timestamp, :completed_weight, -weight)
end
def add_chart_data(timestamp, field, value)
date = timestamp.to_date
date = milestone.start_date if date < milestone.start_date
if chart_data.empty?
chart_data.push({
date: date,
scope_count: 0,
scope_weight: 0,
completed_count: 0,
completed_weight: 0
})
elsif chart_data.last[:date] != date
# To start a new day entry we copy the previous day's data because the numbers are cumulative
chart_data.push(
chart_data.last.merge(date: date)
)
end
chart_data.last[field] += value
end
def find_issue_state(issue_id)
issue_states[issue_id] ||= {
milestone: nil,
weight: 0,
state: ResourceStateEvent.states[:opened]
}
end
# rubocop: disable CodeReuse/ActiveRecord
def resource_events
strong_memoize(:resource_events) do
union = Gitlab::SQL::Union.new([milestone_events, state_events, weight_events]) # rubocop: disable Gitlab/Union
ActiveRecord::Base.connection.execute("(#{union.to_sql}) ORDER BY created_at LIMIT #{EVENT_COUNT_LIMIT + 1}")
end
end
# rubocop: enable CodeReuse/ActiveRecord
def milestone_events
ResourceMilestoneEvent.by_issue_ids_and_created_at_earlier_or_equal_to(issue_ids, end_time)
.select('\'milestone\' AS event_type, created_at, milestone_id AS value, action, issue_id')
end
def state_events
ResourceStateEvent.by_issue_ids_and_created_at_earlier_or_equal_to(issue_ids, end_time)
.select('\'state\' AS event_type, created_at, state AS value, NULL AS action, issue_id')
end
def weight_events
ResourceWeightEvent.by_issue_ids_and_created_at_earlier_or_equal_to(issue_ids, end_time)
.select('\'weight\' AS event_type, created_at, weight AS value, NULL AS action, issue_id')
end
# rubocop: disable CodeReuse/ActiveRecord
def issue_ids
# We find all issues that have this milestone added before this milestone's due date.
# We cannot just filter by `issues.milestone_id` because there might be issues that have
# since been moved to other milestones and we still need to include these in this graph.
ResourceMilestoneEvent
.select(:issue_id)
.where({
milestone_id: milestone.id,
action: :add
})
.where('created_at <= ?', end_time)
end
# rubocop: enable CodeReuse/ActiveRecord
def end_time
@end_time ||= milestone.due_date.end_of_day
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Milestones::BurnupChartService do
let_it_be(:project) { create(:project) }
let_it_be(:milestone) { create(:milestone, project: project, start_date: '2020-01-01', due_date: '2020-01-15') }
let_it_be(:issues) { create_list(:issue, 5, project: project) }
let(:chart_data) { described_class.new(milestone).execute }
it 'raises an error when milestone does not have a start and due date' do
milestone = build(:milestone, project: project)
expect { described_class.new(milestone) }.to raise_error('Milestone must have a start and due date')
end
it 'raises an error when the number of events exceeds the limit' do
stub_const('Milestones::BurnupChartService::EVENT_COUNT_LIMIT', 1)
create(:resource_milestone_event, issue: issues[0], milestone: milestone, action: :add, created_at: '2019-12-15')
create(:resource_milestone_event, issue: issues[1], milestone: milestone, action: :add, created_at: '2019-12-16')
expect { chart_data }.to raise_error(described_class::TooManyEventsError)
end
it 'aggregates events before the start date to the start date' do
create(:resource_milestone_event, issue: issues[0], milestone: milestone, action: :add, created_at: '2019-12-15')
create(:resource_weight_event, issue: issues[0], weight: 2, created_at: '2019-12-18')
create(:resource_milestone_event, issue: issues[1], milestone: milestone, action: :add, created_at: '2019-12-16')
create(:resource_weight_event, issue: issues[1], weight: 1, created_at: '2019-12-18')
create(:resource_milestone_event, issue: issues[2], milestone: milestone, action: :add, created_at: '2019-12-16')
create(:resource_weight_event, issue: issues[2], weight: 3, created_at: '2019-12-18')
create(:resource_state_event, issue: issues[2], state: :closed, created_at: '2019-12-25')
create(:resource_milestone_event, issue: issues[3], milestone: milestone, action: :add, created_at: '2019-12-17')
create(:resource_weight_event, issue: issues[3], weight: 4, created_at: '2019-12-18')
create(:resource_state_event, issue: issues[3], state: :closed, created_at: '2019-12-26')
expect(chart_data).to eq([
{
date: Date.parse('2020-01-01'),
scope_count: 4,
scope_weight: 10,
completed_count: 2,
completed_weight: 7
}
])
end
it 'updates counts and weight when the milestone is added or removed' do
# Add milestone to an open issue with no weight.
create(:resource_milestone_event, issue: issues[0], milestone: milestone, action: :add, created_at: '2020-01-05 03:00')
# Ignore duplicate add event.
create(:resource_milestone_event, issue: issues[0], milestone: milestone, action: :add, created_at: '2020-01-05 03:00')
# Add milestone to an open issue with weight 2 on the same day. This should increment the scope totals for the same day.
create(:resource_weight_event, issue: issues[1], weight: 2, created_at: '2020-01-01')
create(:resource_milestone_event, issue: issues[1], milestone: milestone, action: :add, created_at: '2020-01-05 05:00')
# Add milestone to already closed issue with weight 3. This should increment both the scope and completed totals.
create(:resource_weight_event, issue: issues[2], weight: 3, created_at: '2020-01-01')
create(:resource_state_event, issue: issues[2], state: :closed, created_at: '2020-01-05')
create(:resource_milestone_event, issue: issues[2], milestone: milestone, action: :add, created_at: '2020-01-06')
# Remove milestone from the 2nd open issue. This should decrement the scope totals.
create(:resource_milestone_event, issue: issues[1], milestone: milestone, action: :remove, created_at: '2020-01-07')
# Remove milestone from the closed issue. This should decrement both the scope and completed totals.
create(:resource_milestone_event, issue: issues[2], milestone: milestone, action: :remove, created_at: '2020-01-08')
# Adding a different milestone should not affect the data.
create(:resource_milestone_event, issue: issues[3], milestone: create(:milestone, project: project), action: :add, created_at: '2020-01-08')
# Adding the milestone after the due date should not affect the data.
create(:resource_milestone_event, issue: issues[4], milestone: milestone, action: :add, created_at: '2020-01-30')
# Removing the milestone after the due date should not affect the data.
create(:resource_milestone_event, issue: issues[0], milestone: milestone, action: :remove, created_at: '2020-01-30')
expect(chart_data).to eq([
{
date: Date.parse('2020-01-05'),
scope_count: 2,
scope_weight: 2,
completed_count: 0,
completed_weight: 0
},
{
date: Date.parse('2020-01-06'),
scope_count: 3,
scope_weight: 5,
completed_count: 1,
completed_weight: 3
},
{
date: Date.parse('2020-01-07'),
scope_count: 2,
scope_weight: 3,
completed_count: 1,
completed_weight: 3
},
{
date: Date.parse('2020-01-08'),
scope_count: 1,
scope_weight: 0,
completed_count: 0,
completed_weight: 0
}
])
end
it 'updates the completed counts when issue state is changed' do
# Close an issue assigned to the milestone with weight 2. This should increment the completed totals.
create(:resource_milestone_event, issue: issues[0], milestone: milestone, action: :add, created_at: '2020-01-01 01:00')
create(:resource_weight_event, issue: issues[0], weight: 2, created_at: '2020-01-01 02:00')
create(:resource_state_event, issue: issues[0], state: :closed, created_at: '2020-01-02')
# Closing an issue that is already closed should be ignored.
create(:resource_state_event, issue: issues[0], state: :closed, created_at: '2020-01-03')
# Re-opening the issue should decrement the completed totals.
create(:resource_state_event, issue: issues[0], state: :reopened, created_at: '2020-01-04')
# Closing and re-opening an issue on the same day should not change the totals.
create(:resource_milestone_event, issue: issues[1], milestone: milestone, action: :add, created_at: '2020-01-05 01:00')
create(:resource_weight_event, issue: issues[1], weight: 3, created_at: '2020-01-05 02:00')
create(:resource_state_event, issue: issues[1], state: :closed, created_at: '2020-01-06 05:00')
create(:resource_state_event, issue: issues[1], state: :reopened, created_at: '2020-01-06 08:00')
# Re-opening an issue that is already open should be ignored.
create(:resource_state_event, issue: issues[1], state: :reopened, created_at: '2020-01-07')
# Closing a re-opened issue should increment the completed totals.
create(:resource_state_event, issue: issues[1], state: :closed, created_at: '2020-01-08')
# Changing state when the milestone is already removed should not affect the data.
create(:resource_milestone_event, issue: issues[1], action: :remove, created_at: '2020-01-09')
create(:resource_state_event, issue: issues[1], state: :closed, created_at: '2020-01-10')
expect(chart_data).to eq([
{
date: Date.parse('2020-01-01'),
scope_count: 1,
scope_weight: 2,
completed_count: 0,
completed_weight: 0
},
{
date: Date.parse('2020-01-02'),
scope_count: 1,
scope_weight: 2,
completed_count: 1,
completed_weight: 2
},
{
date: Date.parse('2020-01-04'),
scope_count: 1,
scope_weight: 2,
completed_count: 0,
completed_weight: 0
},
{
date: Date.parse('2020-01-05'),
scope_count: 2,
scope_weight: 5,
completed_count: 0,
completed_weight: 0
},
{
date: Date.parse('2020-01-06'),
scope_count: 2,
scope_weight: 5,
completed_count: 0,
completed_weight: 0
},
{
date: Date.parse('2020-01-08'),
scope_count: 2,
scope_weight: 5,
completed_count: 1,
completed_weight: 3
},
{
date: Date.parse('2020-01-09'),
scope_count: 1,
scope_weight: 2,
completed_count: 0,
completed_weight: 0
}
])
end
it 'updates the weight totals when issue weight is changed' do
# Issue starts out with no weight and should increment once the weight is changed to 2.
create(:resource_milestone_event, issue: issues[0], milestone: milestone, action: :add, created_at: '2020-01-01')
create(:resource_weight_event, issue: issues[0], weight: 2, created_at: '2020-01-02')
# A closed issue is added and weight is set to 5 and should add to the weight totals.
create(:resource_milestone_event, issue: issues[1], milestone: milestone, action: :add, created_at: '2020-01-03 01:00')
create(:resource_state_event, issue: issues[1], state: :closed, created_at: '2020-01-03 02:00')
create(:resource_weight_event, issue: issues[1], weight: 5, created_at: '2020-01-03 03:00')
# Lowering the weight of the 2nd issue should decrement the weight totals.
create(:resource_weight_event, issue: issues[1], weight: 1, created_at: '2020-01-04')
# After the first issue is assigned to another milestone, weight changes shouldn't affect the data.
create(:resource_milestone_event, issue: issues[0], milestone: create(:milestone, project: project), action: :add, created_at: '2020-01-05')
create(:resource_weight_event, issue: issues[0], weight: 10, created_at: '2020-01-06')
expect(chart_data).to eq([
{
date: Date.parse('2020-01-01'),
scope_count: 1,
scope_weight: 0,
completed_count: 0,
completed_weight: 0
},
{
date: Date.parse('2020-01-02'),
scope_count: 1,
scope_weight: 2,
completed_count: 0,
completed_weight: 0
},
{
date: Date.parse('2020-01-03'),
scope_count: 2,
scope_weight: 7,
completed_count: 1,
completed_weight: 5
},
{
date: Date.parse('2020-01-04'),
scope_count: 2,
scope_weight: 3,
completed_count: 1,
completed_weight: 1
},
{
date: Date.parse('2020-01-05'),
scope_count: 1,
scope_weight: 1,
completed_count: 1,
completed_weight: 1
}
])
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment