Show in progress items in VSA

This change implements query changes to the value stream analytics
feature to show and calulate metrics on in progress items.

ee/app/controllers/concerns/ee/cycle_analytics_params.rb

@@ -14,6 +14,7 @@ module EE
           options[:group] = params[:group_id] if params[:group_id]
           options[:from] = params[:from] if params[:from]
           options[:to] = params[:to] if params[:to]
+          options[:end_event_filter] = params[:end_event_filter] if params[:end_event_filter]
           options.merge!(params.slice(*::Gitlab::Analytics::CycleAnalytics::RequestParams::FINDER_PARAM_NAMES))
         end
       end

ee/lib/gitlab/analytics/cycle_analytics/request_params.rb

@@ -21,6 +21,7 @@ module Gitlab
           :direction,
           :page,
           :stage_id,
+          :end_event_filter,
           label_name: [].freeze,
           assignee_username: [].freeze,
           project_ids: [].freeze
@@ -44,6 +45,7 @@ module Gitlab
         attribute :direction
         attribute :page
         attribute :stage_id
+        attribute :end_event_filter
 
         FINDER_PARAM_NAMES.each do |param_name|
           attribute param_name
@@ -60,6 +62,7 @@ module Gitlab
 
           self.created_before = (self.created_before || Time.current).at_end_of_day
           self.created_after = (created_after || default_created_after).at_beginning_of_day
+          self.end_event_filter ||= Gitlab::Analytics::CycleAnalytics::BaseQueryBuilder::DEFAULT_END_EVENT_FILTER
         end
 
         def project_ids
@@ -74,7 +77,8 @@ module Gitlab
             project_ids: project_ids,
             sort: sort&.to_sym,
             direction: direction&.to_sym,
-            page: page
+            page: page,
+            end_event_filter: end_event_filter.to_sym
           }.merge(attributes.symbolize_keys.slice(*FINDER_PARAM_NAMES))
         end
 

ee/lib/gitlab/analytics/cycle_analytics/stage_events/label_based_stage_event.rb

@@ -32,6 +32,12 @@ module Gitlab
           end
           # rubocop: enable CodeReuse/ActiveRecord
 
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_negated_query_customization(query)
+            query.where('NOT EXISTS (?)', subquery)
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
+
           private
 
           def resource_label_events_table

ee/lib/gitlab/analytics/cycle_analytics/summary/group/base_time.rb

@@ -44,6 +44,7 @@ module Gitlab
                   from: @options[:from],
                   to: @options[:to] || DateTime.now,
                   project_ids: @options[:projects],
+                  end_event_filter: @options[:end_event_filter],
                   current_user: @current_user
                 }.merge(@options.slice(*::Gitlab::Analytics::CycleAnalytics::RequestParams::FINDER_PARAM_NAMES))
               )

ee/spec/lib/gitlab/analytics/cycle_analytics/data_collector_spec.rb

@@ -5,8 +5,10 @@ require 'spec_helper'
 RSpec.describe Gitlab::Analytics::CycleAnalytics::DataCollector do
   let_it_be(:user) { create(:user) }
 
+  let(:current_time) { Time.new(2019, 6, 1) }
+
   around do |example|
-    Timecop.freeze { example.run }
+    Timecop.freeze(current_time) { example.run }
   end
 
   def round_to_days(seconds)
@@ -21,11 +23,11 @@ RSpec.describe Gitlab::Analytics::CycleAnalytics::DataCollector do
   shared_examples 'custom Value Stream Analytics Stage' do
     let(:params) { { from: Time.new(2019), to: Time.new(2020), current_user: user } }
     let(:data_collector) { described_class.new(stage: stage, params: params) }
-    let(:resource_1_end_time) { Time.new(2019, 3, 15) }
-    let(:resource_2_end_time) { Time.new(2019, 3, 10) }
-    let(:resource_3_end_time) { Time.new(2019, 3, 20) }
+    let_it_be(:resource_1_end_time) { Time.new(2019, 3, 15) }
+    let_it_be(:resource_2_end_time) { Time.new(2019, 3, 10) }
+    let_it_be(:resource_3_end_time) { Time.new(2019, 3, 20) }
 
-    let!(:resource1) do
+    let_it_be(:resource1) do
       # takes 10 days
       resource = travel_to(Time.new(2019, 3, 5)) do
         create_data_for_start_event(self)
@@ -38,7 +40,7 @@ RSpec.describe Gitlab::Analytics::CycleAnalytics::DataCollector do
       resource
     end
 
-    let!(:resource2) do
+    let_it_be(:resource2) do
       # takes 5 days
       resource = travel_to(Time.new(2019, 3, 5)) do
         create_data_for_start_event(self)
@@ -51,7 +53,7 @@ RSpec.describe Gitlab::Analytics::CycleAnalytics::DataCollector do
       resource
     end
 
-    let!(:resource3) do
+    let_it_be(:resource3) do
       # takes 15 days
       resource = travel_to(Time.new(2019, 3, 5)) do
         create_data_for_start_event(self)
@@ -64,6 +66,21 @@ RSpec.describe Gitlab::Analytics::CycleAnalytics::DataCollector do
       resource
     end
 
+    let_it_be(:unfinished_resource_1_start_time) { Time.new(2019, 3, 5) }
+    let_it_be(:unfinished_resource_2_start_time) { Time.new(2019, 5, 10) }
+
+    let_it_be(:unfinished_resource_1) do
+      travel_to(unfinished_resource_1_start_time) do
+        create_data_for_start_event(self)
+      end
+    end
+
+    let_it_be(:unfinished_resource_2) do
+      travel_to(unfinished_resource_2_start_time) do
+        create_data_for_start_event(self)
+      end
+    end
+
     it 'loads serialized records' do
       items = data_collector.serialized_records
       expect(items.size).to eq(3)
@@ -116,6 +133,32 @@ RSpec.describe Gitlab::Analytics::CycleAnalytics::DataCollector do
 
       it { is_expected.to eq(3) }
     end
+
+    context 'when filtering in progress items' do
+      before do
+        params[:end_event_filter] = :in_progress
+      end
+
+      describe '#count' do
+        subject(:count) { data_collector.count }
+
+        it { is_expected.to eq(2) }
+      end
+
+      it 'calculates median' do
+        duration_1 = current_time - unfinished_resource_1_start_time
+        duration_2 = current_time - unfinished_resource_2_start_time
+
+        expected_median = (duration_1 + duration_2).fdiv(2)
+
+        expect(round_to_days(data_collector.median.seconds)).to eq(round_to_days(expected_median))
+      end
+
+      it 'loads serialized records' do
+        items = data_collector.serialized_records
+        expect(items.size).to eq(2)
+      end
+    end
   end
 
   shared_examples 'test various start and end event combinations' do
@@ -209,11 +252,13 @@ RSpec.describe Gitlab::Analytics::CycleAnalytics::DataCollector do
         def create_data_for_start_event(example_class)
           issue = create(:issue, :opened, project: example_class.project)
 
+          Sidekiq::Worker.skipping_transaction_check do
             Issues::UpdateService.new(
               example_class.project,
               user,
               label_ids: [example_class.label.id]
             ).execute(issue)
+          end
 
           issue
         end
@@ -241,22 +286,26 @@ RSpec.describe Gitlab::Analytics::CycleAnalytics::DataCollector do
         def create_data_for_start_event(example_class)
           issue = create(:issue, :opened, project: example_class.project)
 
+          Sidekiq::Worker.skipping_transaction_check do
             Issues::UpdateService.new(
               example_class.project,
               user,
               label_ids: [example_class.label.id]
             ).execute(issue)
+          end
 
           issue
         end
 
         def create_data_for_end_event(issue, example_class)
+          Sidekiq::Worker.skipping_transaction_check do
             Issues::UpdateService.new(
               example_class.project,
               user,
               label_ids: [example_class.label.id, example_class.other_label.id]
             ).execute(issue)
           end
+        end
 
         it_behaves_like 'custom Value Stream Analytics Stage' do
           context 'when filtering for two labels' do
@@ -291,12 +340,14 @@ RSpec.describe Gitlab::Analytics::CycleAnalytics::DataCollector do
         end
 
         def create_data_for_end_event(issue, example_class)
+          Sidekiq::Worker.skipping_transaction_check do
             Issues::UpdateService.new(
               example_class.project,
               user,
               label_ids: [example_class.label.id]
             ).execute(issue)
           end
+        end
 
         it_behaves_like 'custom Value Stream Analytics Stage'
       end
@@ -411,29 +462,33 @@ RSpec.describe Gitlab::Analytics::CycleAnalytics::DataCollector do
         def create_data_for_start_event(example_class)
           mr = create(:merge_request, source_project: example_class.project, allow_broken: true)
 
+          Sidekiq::Worker.skipping_transaction_check do
             MergeRequests::UpdateService.new(
               example_class.project,
               user,
               label_ids: [label.id]
             ).execute(mr)
+          end
 
           mr
         end
 
         def create_data_for_end_event(mr, example_class)
+          Sidekiq::Worker.skipping_transaction_check do
             MergeRequests::UpdateService.new(
               example_class.project,
               user,
               label_ids: []
             ).execute(mr)
           end
+        end
 
         it_behaves_like 'custom Value Stream Analytics Stage'
       end
 
-      context 'between code stage start time and merge request created time' do
+      context 'between code stage start time and merge request closed time' do
         let(:start_event_identifier) { :code_stage_start }
-        let(:end_event_identifier) { :merge_request_created }
+        let(:end_event_identifier) { :merge_request_closed }
 
         context 'when issue is referenced in the commit message' do
           def create_data_for_start_event(example_class)
@@ -446,18 +501,20 @@ RSpec.describe Gitlab::Analytics::CycleAnalytics::DataCollector do
               allow_broken: true
             })
 
+            Sidekiq::Worker.skipping_transaction_check do
               MergeRequests::UpdateService.new(
                 example_class.project,
                 user,
                 assignees: [user]
               ).execute(mr)
+            end
 
             mr.metrics.update!(first_commit_at: Time.zone.now)
             mr
           end
 
           def create_data_for_end_event(mr, example_class)
-            mr.update!(created_at: Time.zone.now)
+            mr.metrics.update!(latest_closed_at: Time.zone.now)
           end
 
           it_behaves_like 'custom Value Stream Analytics Stage'
@@ -471,7 +528,7 @@ RSpec.describe Gitlab::Analytics::CycleAnalytics::DataCollector do
           end
 
           def create_data_for_end_event(mr, example_class)
-            mr.update!(created_at: Time.zone.now)
+            mr.metrics.update!(latest_closed_at: Time.zone.now)
           end
 
           it_behaves_like 'custom Value Stream Analytics Stage'
@@ -493,17 +550,19 @@ RSpec.describe Gitlab::Analytics::CycleAnalytics::DataCollector do
               allow_broken: true
             })
 
+            Sidekiq::Worker.skipping_transaction_check do
               MergeRequests::UpdateService.new(
                 example_class.project,
                 user,
                 label_ids: [label.id, other_label.id]
               ).execute(mr)
+            end
 
             mr
           end
 
           def create_data_for_end_event(mr, example_class)
-            mr.update!(created_at: Time.zone.now)
+            mr.metrics.update!(latest_closed_at: Time.zone.now)
           end
 
           it_behaves_like 'custom Value Stream Analytics Stage'

ee/spec/support/shared_examples/controllers/analytics/cycle_analytics/shared_stage_shared_examples.rb

@@ -227,7 +227,9 @@ RSpec.shared_examples 'Value Stream Analytics Stages controller' do
         end
 
         it 'accepts sort params' do
-          expect(Gitlab::Analytics::CycleAnalytics::Sorting).to receive(:apply).with(kind_of(ActiveRecord::Relation), kind_of(Analytics::CycleAnalytics::GroupStage), :duration, :asc).and_call_original
+          expect_next_instance_of(Gitlab::Analytics::CycleAnalytics::Sorting) do |sort|
+            expect(sort).to receive(:apply).with(:duration, :asc).and_call_original
+          end
 
           subject
 

lib/gitlab/analytics/cycle_analytics/average.rb

@@ -7,9 +7,10 @@ module Gitlab
         include Gitlab::Utils::StrongMemoize
         include StageQueryHelpers
 
-        def initialize(stage:, query:)
+        def initialize(stage:, query:, params: {})
           @stage = stage
           @query = query
+          @params = params
         end
 
         def seconds
@@ -22,7 +23,7 @@ module Gitlab
 
         private
 
-        attr_reader :stage
+        attr_reader :stage, :params
 
         # rubocop: disable CodeReuse/ActiveRecord
         def select_average

lib/gitlab/analytics/cycle_analytics/base_query_builder.rb

@@ -5,6 +5,7 @@ module Gitlab
     module CycleAnalytics
       class BaseQueryBuilder
         include Gitlab::CycleAnalytics::MetricsTables
+        include StageQueryHelpers
 
         delegate :subject_class, to: :stage
 
@@ -13,6 +14,8 @@ module Gitlab
           Issue.to_s => IssuesFinder
         }.freeze
 
+        DEFAULT_END_EVENT_FILTER = :finished
+
         def initialize(stage:, params: {})
           @stage = stage
           @params = build_finder_params(params)
@@ -22,8 +25,7 @@ module Gitlab
         def build
           query = finder.execute
           query = stage.start_event.apply_query_customization(query)
-          query = stage.end_event.apply_query_customization(query)
-          query.where(duration_condition)
+          apply_end_event_query_customization(query)
         end
         # rubocop: enable CodeReuse/ActiveRecord
 
@@ -46,6 +48,7 @@ module Gitlab
         def build_finder_params(params)
           {}.tap do |finder_params|
             finder_params[:current_user] = params[:current_user]
+            finder_params[:end_event_filter] = params[:end_event_filter] || DEFAULT_END_EVENT_FILTER
 
             add_parent_model_params!(finder_params)
             add_time_range_params!(finder_params, params[:from], params[:to])
@@ -62,6 +65,17 @@ module Gitlab
           finder_params[:created_after] = from || 30.days.ago
           finder_params[:created_before] = to if to
         end
+
+        # rubocop: disable CodeReuse/ActiveRecord
+        def apply_end_event_query_customization(query)
+          if in_progress?
+            stage.end_event.apply_negated_query_customization(query)
+          else
+            query = stage.end_event.apply_query_customization(query)
+            query.where(duration_condition)
+          end
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
       end
     end
   end

lib/gitlab/analytics/cycle_analytics/data_collector.rb

@@ -29,13 +29,13 @@ module Gitlab
 
         def median
           strong_memoize(:median) do
-            Median.new(stage: stage, query: query)
+            Median.new(stage: stage, query: query, params: params)
           end
         end
 
         def average
           strong_memoize(:average) do
-            Average.new(stage: stage, query: query)
+            Average.new(stage: stage, query: query, params: params)
           end
         end
 

lib/gitlab/analytics/cycle_analytics/median.rb

@@ -6,9 +6,10 @@ module Gitlab
       class Median
         include StageQueryHelpers
 
-        def initialize(stage:, query:)
+        def initialize(stage:, query:, params: {})
           @stage = stage
           @query = query
+          @params = params
         end
 
         # rubocop: disable CodeReuse/ActiveRecord
@@ -26,7 +27,7 @@ module Gitlab
 
         private
 
-        attr_reader :stage
+        attr_reader :stage, :params
 
         def percentile_cont
           percentile_cont_ordering = Arel::Nodes::UnaryOperation.new(Arel::Nodes::SqlLiteral.new('ORDER BY'), duration)

lib/gitlab/analytics/cycle_analytics/records_fetcher.rb

@@ -124,7 +124,7 @@ module Gitlab
         def time_columns
           [
             stage.start_event.timestamp_projection.as('start_event_timestamp'),
-            stage.end_event.timestamp_projection.as('end_event_timestamp'),
+            end_event_timestamp_projection.as('end_event_timestamp'),
             round_duration_to_seconds.as('total_time')
           ]
         end

lib/gitlab/analytics/cycle_analytics/sorting.rb

@@ -4,23 +4,35 @@ module Gitlab
   module Analytics
     module CycleAnalytics
       class Sorting
+        include StageQueryHelpers
+
+        def initialize(stage:, query:, params: {})
+          @stage = stage
+          @query = query
+          @params = params
+        end
+
         # rubocop: disable CodeReuse/ActiveRecord
-        SORTING_OPTIONS = {
+        def apply(sort, direction)
+          sorting_options = {
             end_event: {
-            asc: -> (query, stage) { query.reorder(stage.end_event.timestamp_projection.asc) },
-            desc: -> (query, stage) { query.reorder(stage.end_event.timestamp_projection.desc) }
-          }.freeze,
+              asc: -> { query.reorder(end_event_timestamp_projection.asc) },
+              desc: -> { query.reorder(end_event_timestamp_projection.desc) }
+            },
             duration: {
-            asc: -> (query, stage) { query.reorder(Arel::Nodes::Subtraction.new(stage.end_event.timestamp_projection, stage.start_event.timestamp_projection).asc) },
-            desc: -> (query, stage) { query.reorder(Arel::Nodes::Subtraction.new(stage.end_event.timestamp_projection, stage.start_event.timestamp_projection).desc) }
-          }.freeze
-        }.freeze
-        # rubocop: enable CodeReuse/ActiveRecord,
+              asc: -> { query.reorder(duration.asc) },
+              desc: -> { query.reorder(duration.desc) }
+            }
+          }
 
-        def self.apply(query, stage, sort, direction)
-          sort_lambda = SORTING_OPTIONS.dig(sort, direction) || SORTING_OPTIONS.dig(:end_event, :desc)
-          sort_lambda.call(query, stage)
+          sort_lambda = sorting_options.dig(sort, direction) || sorting_options.dig(:end_event, :desc)
+          sort_lambda.call
         end
+        # rubocop: enable CodeReuse/ActiveRecord
+
+        private
+
+        attr_reader :stage, :query, :params
       end
     end
   end

lib/gitlab/analytics/cycle_analytics/stage_events/metrics_based_stage_event.rb

@@ -11,6 +11,12 @@ module Gitlab
           end
           # rubocop: enable CodeReuse/ActiveRecord
 
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_negated_query_customization(query)
+            super.joins(:metrics)
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
+
           def column_list
             [timestamp_projection]
           end

lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb

@@ -51,6 +51,12 @@ module Gitlab
             query
           end
 
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_negated_query_customization(query)
+            query.where(timestamp_projection.eq(nil))
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
+
           def self.label_based?
             false
           end

lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb

@@ -18,22 +18,30 @@ module Gitlab
 
         def duration
           Arel::Nodes::Subtraction.new(
-            stage.end_event.timestamp_projection,
+            end_event_timestamp_projection,
             stage.start_event.timestamp_projection
           )
         end
 
+        def end_event_timestamp_projection
+          if in_progress?
+            Arel::Nodes::NamedFunction.new('TO_TIMESTAMP', [Time.current.to_i])
+          else
+            stage.end_event.timestamp_projection
+          end
+        end
+
         # rubocop: disable CodeReuse/ActiveRecord
         def order_by(query, sort, direction, extra_columns_to_select = [:id])
-          ordered_query = Gitlab::Analytics::CycleAnalytics::Sorting.apply(query, stage, sort, direction)
+          ordered_query = Gitlab::Analytics::CycleAnalytics::Sorting.new(stage: stage, query: query, params: params).apply(sort, direction)
 
           # When filtering for more than one label, postgres requires the columns in ORDER BY to be present in the GROUP BY clause
           if requires_grouping?
-            column_list = [
-              *extra_columns_to_select,
-              *stage.end_event.column_list,
-              *stage.start_event.column_list
-            ]
+            column_list = [].tap do |array|
+              array.concat(extra_columns_to_select)
+              array.concat(stage.end_event.column_list) unless in_progress?
+              array.concat(stage.start_event.column_list)
+            end
 
             ordered_query = ordered_query.group(column_list)
           end
@@ -45,6 +53,10 @@ module Gitlab
         def requires_grouping?
           Array(params[:label_name]).size > 1
         end
+
+        def in_progress?
+          params[:end_event_filter] == :in_progress
+        end
       end
     end
   end

spec/lib/gitlab/analytics/cycle_analytics/sorting_spec.rb

@@ -5,7 +5,7 @@ require 'spec_helper'
 RSpec.describe Gitlab::Analytics::CycleAnalytics::Sorting do
   let(:stage) { build(:cycle_analytics_project_stage, start_event_identifier: :merge_request_created, end_event_identifier: :merge_request_merged) }
 
-  subject(:order_values) { described_class.apply(MergeRequest.joins(:metrics), stage, sort, direction).order_values }
+  subject(:order_values) { described_class.new(query: MergeRequest.joins(:metrics), stage: stage).apply(sort, direction).order_values }
 
   context 'when invalid sorting params are given' do
     let(:sort) { :unknown_sort }