Merge branch 'milestone-burnup-charts-backend' into 'master'

Create a service to compute burnup charts for milestones

See merge request gitlab-org/gitlab!37636

changelogs/unreleased/milestone-burnup-charts-backend.yml

+---
+title: Add index to resource_milestone_events for add actions
+merge_request: 37636
+author:
+type: other

db/migrate/20200729181641_add_index_to_resource_milestone_events_add_events.rb

+# frozen_string_literal: true
+
+class AddIndexToResourceMilestoneEventsAddEvents < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  INDEX_NAME = 'index_resource_milestone_events_on_milestone_id_and_add_action'
+  ADD_ACTION = '1'
+
+  def up
+    # Index add milestone events
+    add_concurrent_index :resource_milestone_events, :milestone_id, where: "action = #{ADD_ACTION}", name: INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index :resource_milestone_events, :milestone_id, name: INDEX_NAME
+  end
+end

db/schema_migrations/20200729181641

+5dc4cbfc6d7e79e5909e5250f382bc3c9fa4246b8f2aed81404899aee4eef81b
\ No newline at end of file

db/structure.sql

@@ -20468,6 +20468,8 @@ CREATE INDEX index_resource_milestone_events_on_merge_request_id ON public.resou
 
 CREATE INDEX index_resource_milestone_events_on_milestone_id ON public.resource_milestone_events USING btree (milestone_id);
 
+CREATE INDEX index_resource_milestone_events_on_milestone_id_and_add_action ON public.resource_milestone_events USING btree (milestone_id) WHERE (action = 1);
+
 CREATE INDEX index_resource_milestone_events_on_user_id ON public.resource_milestone_events USING btree (user_id);
 
 CREATE INDEX index_resource_state_events_on_epic_id ON public.resource_state_events USING btree (epic_id);

ee/app/services/milestones/burnup_chart_service.rb

+# frozen_string_literal: true
+
+# This service computes the milestone's daily total number of issues and their weights.
+# For each day, this returns the totals for all issues that are assigned to the milestone at that point in time.
+# This represents the scope for this milestone. This also returns separate totals for closed issues which represent the completed work.
+#
+# This is implemented by iterating over all relevant resource events ordered by time. We need to do this
+# so that we can keep track of the issue's state during that point in time and handle the events based on that.
+
+class Milestones::BurnupChartService
+  include Gitlab::Utils::StrongMemoize
+
+  EVENT_COUNT_LIMIT = 50_000
+
+  TooManyEventsError = Class.new(StandardError)
+
+  def initialize(milestone)
+    raise ArgumentError, 'Milestone must have a start and due date' if milestone.start_date.blank? || milestone.due_date.blank?
+
+    @milestone = milestone
+  end
+
+  def execute
+    @issue_states = {}
+    @chart_data = []
+
+    raise TooManyEventsError if resource_events.num_tuples > EVENT_COUNT_LIMIT
+
+    resource_events.each do |event|
+      case event['event_type']
+      when 'milestone'
+        handle_milestone_event(event)
+      when 'state'
+        handle_state_event(event)
+      when 'weight'
+        handle_weight_event(event)
+      end
+    end
+
+    chart_data
+  end
+
+  private
+
+  attr_reader :milestone, :issue_states, :chart_data
+
+  def handle_milestone_event(event)
+    issue_state = find_issue_state(event['issue_id'])
+
+    return if issue_state[:milestone] == milestone.id && event['action'] == ResourceMilestoneEvent.actions[:add] && event['value'] == milestone.id
+
+    if event['action'] == ResourceMilestoneEvent.actions[:add] && event['value'] == milestone.id
+      handle_add_milestone_event(event)
+    elsif issue_state[:milestone] == milestone.id
+      # If the issue is currently assigned to the milestone, then treat any event here as a removal.
+      # We do not have a separate `:remove` event when replacing milestones with another one.
+      handle_remove_milestone_event(event)
+    end
+
+    issue_state[:milestone] = event['value']
+  end
+
+  def handle_add_milestone_event(event)
+    issue_state = find_issue_state(event['issue_id'])
+
+    increment_scope(event['created_at'], issue_state[:weight])
+
+    if issue_state[:state] == ResourceStateEvent.states[:closed]
+      increment_completed(event['created_at'], issue_state[:weight])
+    end
+  end
+
+  def handle_remove_milestone_event(event)
+    issue_state = find_issue_state(event['issue_id'])
+
+    decrement_scope(event['created_at'], issue_state[:weight])
+
+    if issue_state[:state] == ResourceStateEvent.states[:closed]
+      decrement_completed(event['created_at'], issue_state[:weight])
+    end
+  end
+
+  def handle_state_event(event)
+    issue_state = find_issue_state(event['issue_id'])
+    old_state = issue_state[:state]
+    issue_state[:state] = event['value']
+
+    return if issue_state[:milestone] != milestone.id
+
+    if old_state == ResourceStateEvent.states[:closed] && event['value'] == ResourceStateEvent.states[:reopened]
+      decrement_completed(event['created_at'], issue_state[:weight])
+    elsif ResourceStateEvent.states.values_at(:opened, :reopened).include?(old_state) && event['value'] == ResourceStateEvent.states[:closed]
+      increment_completed(event['created_at'], issue_state[:weight])
+    end
+  end
+
+  def handle_weight_event(event)
+    issue_state = find_issue_state(event['issue_id'])
+    old_weight = issue_state[:weight]
+    issue_state[:weight] = event['value'] || 0
+
+    return if issue_state[:milestone] != milestone.id
+
+    add_chart_data(event['created_at'], :scope_weight, issue_state[:weight] - old_weight)
+
+    if issue_state[:state] == ResourceStateEvent.states[:closed]
+      add_chart_data(event['created_at'], :completed_weight, issue_state[:weight] - old_weight)
+    end
+  end
+
+  def increment_scope(timestamp, weight)
+    add_chart_data(timestamp, :scope_count, 1)
+    add_chart_data(timestamp, :scope_weight, weight)
+  end
+
+  def decrement_scope(timestamp, weight)
+    add_chart_data(timestamp, :scope_count, -1)
+    add_chart_data(timestamp, :scope_weight, -weight)
+  end
+
+  def increment_completed(timestamp, weight)
+    add_chart_data(timestamp, :completed_count, 1)
+    add_chart_data(timestamp, :completed_weight, weight)
+  end
+
+  def decrement_completed(timestamp, weight)
+    add_chart_data(timestamp, :completed_count, -1)
+    add_chart_data(timestamp, :completed_weight, -weight)
+  end
+
+  def add_chart_data(timestamp, field, value)
+    date = timestamp.to_date
+    date = milestone.start_date if date < milestone.start_date
+
+    if chart_data.empty?
+      chart_data.push({
+        date: date,
+        scope_count: 0,
+        scope_weight: 0,
+        completed_count: 0,
+        completed_weight: 0
+      })
+    elsif chart_data.last[:date] != date
+      # To start a new day entry we copy the previous day's data because the numbers are cumulative
+      chart_data.push(
+        chart_data.last.merge(date: date)
+      )
+    end
+
+    chart_data.last[field] += value
+  end
+
+  def find_issue_state(issue_id)
+    issue_states[issue_id] ||= {
+      milestone: nil,
+      weight: 0,
+      state: ResourceStateEvent.states[:opened]
+    }
+  end
+
+  # rubocop: disable CodeReuse/ActiveRecord
+  def resource_events
+    strong_memoize(:resource_events) do
+      union = Gitlab::SQL::Union.new([milestone_events, state_events, weight_events]) # rubocop: disable Gitlab/Union
+
+      ActiveRecord::Base.connection.execute("(#{union.to_sql}) ORDER BY created_at LIMIT #{EVENT_COUNT_LIMIT + 1}")
+    end
+  end
+  # rubocop: enable CodeReuse/ActiveRecord
+
+  def milestone_events
+    ResourceMilestoneEvent.by_issue_ids_and_created_at_earlier_or_equal_to(issue_ids, end_time)
+      .select('\'milestone\' AS event_type, created_at, milestone_id AS value, action, issue_id')
+  end
+
+  def state_events
+    ResourceStateEvent.by_issue_ids_and_created_at_earlier_or_equal_to(issue_ids, end_time)
+      .select('\'state\' AS event_type, created_at, state AS value, NULL AS action, issue_id')
+  end
+
+  def weight_events
+    ResourceWeightEvent.by_issue_ids_and_created_at_earlier_or_equal_to(issue_ids, end_time)
+      .select('\'weight\' AS event_type, created_at, weight AS value, NULL AS action, issue_id')
+  end
+
+  # rubocop: disable CodeReuse/ActiveRecord
+  def issue_ids
+    # We find all issues that have this milestone added before this milestone's due date.
+    # We cannot just filter by `issues.milestone_id` because there might be issues that have
+    # since been moved to other milestones and we still need to include these in this graph.
+    ResourceMilestoneEvent
+      .select(:issue_id)
+      .where({
+        milestone_id: milestone.id,
+        action: :add
+      })
+      .where('created_at <= ?', end_time)
+  end
+  # rubocop: enable CodeReuse/ActiveRecord
+
+  def end_time
+    @end_time ||= milestone.due_date.end_of_day
+  end
+end

ee/spec/services/milestones/burnup_chart_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Milestones::BurnupChartService do
+  let_it_be(:project) { create(:project) }
+  let_it_be(:milestone) { create(:milestone, project: project, start_date: '2020-01-01', due_date: '2020-01-15') }
+
+  let_it_be(:issues) { create_list(:issue, 5, project: project) }
+
+  let(:chart_data) { described_class.new(milestone).execute }
+
+  it 'raises an error when milestone does not have a start and due date' do
+    milestone = build(:milestone, project: project)
+
+    expect { described_class.new(milestone) }.to raise_error('Milestone must have a start and due date')
+  end
+
+  it 'raises an error when the number of events exceeds the limit' do
+    stub_const('Milestones::BurnupChartService::EVENT_COUNT_LIMIT', 1)
+
+    create(:resource_milestone_event, issue: issues[0], milestone: milestone, action: :add, created_at: '2019-12-15')
+    create(:resource_milestone_event, issue: issues[1], milestone: milestone, action: :add, created_at: '2019-12-16')
+
+    expect { chart_data }.to raise_error(described_class::TooManyEventsError)
+  end
+
+  it 'aggregates events before the start date to the start date' do
+    create(:resource_milestone_event, issue: issues[0], milestone: milestone, action: :add, created_at: '2019-12-15')
+    create(:resource_weight_event, issue: issues[0], weight: 2, created_at: '2019-12-18')
+
+    create(:resource_milestone_event, issue: issues[1], milestone: milestone, action: :add, created_at: '2019-12-16')
+    create(:resource_weight_event, issue: issues[1], weight: 1, created_at: '2019-12-18')
+
+    create(:resource_milestone_event, issue: issues[2], milestone: milestone, action: :add, created_at: '2019-12-16')
+    create(:resource_weight_event, issue: issues[2], weight: 3, created_at: '2019-12-18')
+    create(:resource_state_event, issue: issues[2], state: :closed, created_at: '2019-12-25')
+
+    create(:resource_milestone_event, issue: issues[3], milestone: milestone, action: :add, created_at: '2019-12-17')
+    create(:resource_weight_event, issue: issues[3], weight: 4, created_at: '2019-12-18')
+    create(:resource_state_event, issue: issues[3], state: :closed, created_at: '2019-12-26')
+
+    expect(chart_data).to eq([
+      {
+        date: Date.parse('2020-01-01'),
+        scope_count: 4,
+        scope_weight: 10,
+        completed_count: 2,
+        completed_weight: 7
+      }
+    ])
+  end
+
+  it 'updates counts and weight when the milestone is added or removed' do
+    # Add milestone to an open issue with no weight.
+    create(:resource_milestone_event, issue: issues[0], milestone: milestone, action: :add, created_at: '2020-01-05 03:00')
+    # Ignore duplicate add event.
+    create(:resource_milestone_event, issue: issues[0], milestone: milestone, action: :add, created_at: '2020-01-05 03:00')
+
+    # Add milestone to an open issue with weight 2 on the same day. This should increment the scope totals for the same day.
+    create(:resource_weight_event, issue: issues[1], weight: 2, created_at: '2020-01-01')
+    create(:resource_milestone_event, issue: issues[1], milestone: milestone, action: :add, created_at: '2020-01-05 05:00')
+
+    # Add milestone to already closed issue with weight 3. This should increment both the scope and completed totals.
+    create(:resource_weight_event, issue: issues[2], weight: 3, created_at: '2020-01-01')
+    create(:resource_state_event, issue: issues[2], state: :closed, created_at: '2020-01-05')
+    create(:resource_milestone_event, issue: issues[2], milestone: milestone, action: :add, created_at: '2020-01-06')
+
+    # Remove milestone from the 2nd open issue. This should decrement the scope totals.
+    create(:resource_milestone_event, issue: issues[1], milestone: milestone, action: :remove, created_at: '2020-01-07')
+
+    # Remove milestone from the closed issue. This should decrement both the scope and completed totals.
+    create(:resource_milestone_event, issue: issues[2], milestone: milestone, action: :remove, created_at: '2020-01-08')
+
+    # Adding a different milestone should not affect the data.
+    create(:resource_milestone_event, issue: issues[3], milestone: create(:milestone, project: project), action: :add, created_at: '2020-01-08')
+
+    # Adding the milestone after the due date should not affect the data.
+    create(:resource_milestone_event, issue: issues[4], milestone: milestone, action: :add, created_at: '2020-01-30')
+
+    # Removing the milestone after the due date should not affect the data.
+    create(:resource_milestone_event, issue: issues[0], milestone: milestone, action: :remove, created_at: '2020-01-30')
+
+    expect(chart_data).to eq([
+      {
+        date: Date.parse('2020-01-05'),
+        scope_count: 2,
+        scope_weight: 2,
+        completed_count: 0,
+        completed_weight: 0
+      },
+      {
+        date: Date.parse('2020-01-06'),
+        scope_count: 3,
+        scope_weight: 5,
+        completed_count: 1,
+        completed_weight: 3
+      },
+      {
+        date: Date.parse('2020-01-07'),
+        scope_count: 2,
+        scope_weight: 3,
+        completed_count: 1,
+        completed_weight: 3
+      },
+      {
+        date: Date.parse('2020-01-08'),
+        scope_count: 1,
+        scope_weight: 0,
+        completed_count: 0,
+        completed_weight: 0
+      }
+    ])
+  end
+
+  it 'updates the completed counts when issue state is changed' do
+    # Close an issue assigned to the milestone with weight 2. This should increment the completed totals.
+    create(:resource_milestone_event, issue: issues[0], milestone: milestone, action: :add, created_at: '2020-01-01 01:00')
+    create(:resource_weight_event, issue: issues[0], weight: 2, created_at: '2020-01-01 02:00')
+    create(:resource_state_event, issue: issues[0], state: :closed, created_at: '2020-01-02')
+
+    # Closing an issue that is already closed should be ignored.
+    create(:resource_state_event, issue: issues[0], state: :closed, created_at: '2020-01-03')
+
+    # Re-opening the issue should decrement the completed totals.
+    create(:resource_state_event, issue: issues[0], state: :reopened, created_at: '2020-01-04')
+
+    # Closing and re-opening an issue on the same day should not change the totals.
+    create(:resource_milestone_event, issue: issues[1], milestone: milestone, action: :add, created_at: '2020-01-05 01:00')
+    create(:resource_weight_event, issue: issues[1], weight: 3, created_at: '2020-01-05 02:00')
+    create(:resource_state_event, issue: issues[1], state: :closed, created_at: '2020-01-06 05:00')
+    create(:resource_state_event, issue: issues[1], state: :reopened, created_at: '2020-01-06 08:00')
+
+    # Re-opening an issue that is already open should be ignored.
+    create(:resource_state_event, issue: issues[1], state: :reopened, created_at: '2020-01-07')
+
+    # Closing a re-opened issue should increment the completed totals.
+    create(:resource_state_event, issue: issues[1], state: :closed, created_at: '2020-01-08')
+
+    # Changing state when the milestone is already removed should not affect the data.
+    create(:resource_milestone_event, issue: issues[1], action: :remove, created_at: '2020-01-09')
+    create(:resource_state_event, issue: issues[1], state: :closed, created_at: '2020-01-10')
+
+    expect(chart_data).to eq([
+      {
+        date: Date.parse('2020-01-01'),
+        scope_count: 1,
+        scope_weight: 2,
+        completed_count: 0,
+        completed_weight: 0
+      },
+      {
+        date: Date.parse('2020-01-02'),
+        scope_count: 1,
+        scope_weight: 2,
+        completed_count: 1,
+        completed_weight: 2
+      },
+      {
+        date: Date.parse('2020-01-04'),
+        scope_count: 1,
+        scope_weight: 2,
+        completed_count: 0,
+        completed_weight: 0
+      },
+      {
+        date: Date.parse('2020-01-05'),
+        scope_count: 2,
+        scope_weight: 5,
+        completed_count: 0,
+        completed_weight: 0
+      },
+      {
+        date: Date.parse('2020-01-06'),
+        scope_count: 2,
+        scope_weight: 5,
+        completed_count: 0,
+        completed_weight: 0
+      },
+      {
+        date: Date.parse('2020-01-08'),
+        scope_count: 2,
+        scope_weight: 5,
+        completed_count: 1,
+        completed_weight: 3
+      },
+      {
+        date: Date.parse('2020-01-09'),
+        scope_count: 1,
+        scope_weight: 2,
+        completed_count: 0,
+        completed_weight: 0
+      }
+    ])
+  end
+
+  it 'updates the weight totals when issue weight is changed' do
+    # Issue starts out with no weight and should increment once the weight is changed to 2.
+    create(:resource_milestone_event, issue: issues[0], milestone: milestone, action: :add, created_at: '2020-01-01')
+    create(:resource_weight_event, issue: issues[0], weight: 2, created_at: '2020-01-02')
+
+    # A closed issue is added and weight is set to 5 and should add to the weight totals.
+    create(:resource_milestone_event, issue: issues[1], milestone: milestone, action: :add, created_at: '2020-01-03 01:00')
+    create(:resource_state_event, issue: issues[1], state: :closed, created_at: '2020-01-03 02:00')
+    create(:resource_weight_event, issue: issues[1], weight: 5, created_at: '2020-01-03 03:00')
+
+    # Lowering the weight of the 2nd issue should decrement the weight totals.
+    create(:resource_weight_event, issue: issues[1], weight: 1, created_at: '2020-01-04')
+
+    # After the first issue is assigned to another milestone, weight changes shouldn't affect the data.
+    create(:resource_milestone_event, issue: issues[0], milestone: create(:milestone, project: project), action: :add, created_at: '2020-01-05')
+    create(:resource_weight_event, issue: issues[0], weight: 10, created_at: '2020-01-06')
+
+    expect(chart_data).to eq([
+      {
+        date: Date.parse('2020-01-01'),
+        scope_count: 1,
+        scope_weight: 0,
+        completed_count: 0,
+        completed_weight: 0
+      },
+      {
+        date: Date.parse('2020-01-02'),
+        scope_count: 1,
+        scope_weight: 2,
+        completed_count: 0,
+        completed_weight: 0
+      },
+      {
+        date: Date.parse('2020-01-03'),
+        scope_count: 2,
+        scope_weight: 7,
+        completed_count: 1,
+        completed_weight: 5
+      },
+      {
+        date: Date.parse('2020-01-04'),
+        scope_count: 2,
+        scope_weight: 3,
+        completed_count: 1,
+        completed_weight: 1
+      },
+      {
+        date: Date.parse('2020-01-05'),
+        scope_count: 1,
+        scope_weight: 1,
+        completed_count: 1,
+        completed_weight: 1
+      }
+    ])
+  end
+end