Merge branch...

Merge branch '241686-add-automatically-counters-for-events-in-know_events-yml-to-usage-data' into 'master'

Add automatically counters for events in know_events.yml to usage data

See merge request gitlab-org/gitlab!40714

lib/gitlab/usage_data.rb

@@ -39,6 +39,7 @@ module Gitlab
             .merge(analytics_unique_visits_data)
             .merge(compliance_unique_visits_data)
             .merge(search_unique_visits_data)
+            .merge(redis_hll_counters)
         end
       end
 
@@ -618,6 +619,10 @@ module Gitlab
         {}
       end
 
+      def redis_hll_counters
+        { redis_hll_counters: ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events_data }
+      end
+
       def analytics_unique_visits_data
         results = ::Gitlab::Analytics::UniqueVisits.analytics_events.each_with_object({}) do |target, hash|
           hash[target] = redis_usage_data { unique_visit_service.unique_visits_for(targets: target) }

lib/gitlab/usage_data_counters/hll_redis_counter.rb

@@ -3,6 +3,8 @@
 module Gitlab
   module UsageDataCounters
     module HLLRedisCounter
+      include Gitlab::Utils::UsageData
+
       DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH = 6.weeks
       DEFAULT_DAILY_KEY_EXPIRY_LENGTH = 29.days
       DEFAULT_REDIS_SLOT = ''.freeze
@@ -53,14 +55,44 @@ module Gitlab
           Gitlab::Redis::HLL.count(keys: keys)
         end
 
+        def categories
+          @categories ||= known_events.map { |event| event[:category] }.uniq
+        end
+
         # @param category [String] the category name
         # @return [Array<String>] list of event names for given category
         def events_for_category(category)
           known_events.select { |event| event[:category] == category.to_s }.map { |event| event[:name] }
         end
 
+        def unique_events_data
+          categories.each_with_object({}) do |category, category_results|
+            events_names = events_for_category(category)
+
+            event_results = events_names.each_with_object({}) do |event, hash|
+              hash[event] = unique_events(event_names: event, start_date: 7.days.ago.to_date, end_date: Date.current)
+            end
+
+            if eligible_for_totals?(events_names)
+              event_results["#{category}_total_unique_counts_weekly"] = unique_events(event_names: events_names, start_date: 7.days.ago.to_date, end_date: Date.current)
+              event_results["#{category}_total_unique_counts_monthly"] = unique_events(event_names: events_names, start_date: 4.weeks.ago.to_date, end_date: Date.current)
+            end
+
+            category_results["#{category}"] = event_results
+          end
+        end
+
         private
 
+        # Allow to add totals for events that are in the same redis slot, category and have the same aggregation level
+        # and if there are more than 1 event
+        def eligible_for_totals?(events_names)
+          return false if events_names.size <= 1
+
+          events = events_for(events_names)
+          events_in_same_slot?(events) && events_in_same_category?(events) && events_same_aggregation?(events)
+        end
+
         def keys_for_aggregation(aggregation, events:, start_date:, end_date:)
           if aggregation.to_sym == :daily
             daily_redis_keys(events: events, start_date: start_date, end_date: end_date)
@@ -78,8 +110,11 @@ module Gitlab
         end
 
         def events_in_same_slot?(events)
+          # if we check one event then redis_slot is only one to check
+          return true if events.size == 1
+
           slot = events.first[:redis_slot]
-          events.all? { |event| event[:redis_slot] == slot }
+          events.all? { |event| event[:redis_slot].present? && event[:redis_slot] == slot }
         end
 
         def events_in_same_category?(events)

spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb

@@ -8,24 +8,46 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s
   let(:entity3) { '34rfjuuy-ce56-sa35-ds34-dfer567dfrf2' }
   let(:entity4) { '8b9a2671-2abf-4bec-a682-22f6a8f7bf31' }
 
+  around do |example|
+    # We need to freeze to a reference time
+    # because visits are grouped by the week number in the year
+    # Without freezing the time, the test may behave inconsistently
+    # depending on which day of the week test is run.
+    # Monday 6th of June
+    reference_time = Time.utc(2020, 6, 1)
+    Timecop.freeze(reference_time) { example.run }
+  end
+
+  describe '.categories' do
+    it 'gets all unique category names' do
+      expect(described_class.categories).to contain_exactly('analytics', 'compliance', 'ide_edit', 'search')
+    end
+  end
+
+  describe 'known_events' do
     let(:weekly_event) { 'g_analytics_contribution' }
     let(:daily_event) { 'g_analytics_search' }
     let(:analytics_slot_event) { 'g_analytics_contribution' }
     let(:compliance_slot_event) { 'g_compliance_dashboard' }
-  let(:category_analytics) { 'g_analytics_search' }
-  let(:category_productivity) { 'g_analytics_productivity' }
+    let(:category_analytics_event) { 'g_analytics_search' }
+    let(:category_productivity_event) { 'g_analytics_productivity' }
     let(:no_slot) { 'no_slot' }
     let(:different_aggregation) { 'different_aggregation' }
     let(:custom_daily_event) { 'g_analytics_custom' }
 
+    let(:global_category) { 'global' }
+    let(:compliance_category) {'compliance' }
+    let(:productivity_category) {'productivity' }
+    let(:analytics_category) { 'analytics' }
+
     let(:known_events) do
       [
-      { name: weekly_event, redis_slot: "analytics", category: "analytics", expiry: 84, aggregation: "weekly" },
-      { name: daily_event, redis_slot: "analytics", category: "analytics", expiry: 84, aggregation: "daily" },
-      { name: category_productivity, redis_slot: "analytics", category: "productivity", aggregation: "weekly" },
-      { name: compliance_slot_event, redis_slot: "compliance", category: "compliance", aggregation: "weekly" },
-      { name: no_slot, category: "global", aggregation: "daily" },
-      { name: different_aggregation, category: "global", aggregation: "monthly" }
+        { name: weekly_event, redis_slot: "analytics", category: analytics_category, expiry: 84, aggregation: "weekly" },
+        { name: daily_event, redis_slot: "analytics", category: analytics_category, expiry: 84, aggregation: "daily" },
+        { name: category_productivity_event, redis_slot: "analytics", category: productivity_category, aggregation: "weekly" },
+        { name: compliance_slot_event, redis_slot: "compliance", category: compliance_category, aggregation: "weekly" },
+        { name: no_slot, category: global_category, aggregation: "daily" },
+        { name: different_aggregation, category: global_category, aggregation: "monthly" }
       ].map(&:with_indifferent_access)
     end
 
@@ -33,16 +55,6 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s
       allow(described_class).to receive(:known_events).and_return(known_events)
     end
 
-  around do |example|
-    # We need to freeze to a reference time
-    # because visits are grouped by the week number in the year
-    # Without freezing the time, the test may behave inconsistently
-    # depending on which day of the week test is run.
-    # Monday 6th of June
-    reference_time = Time.utc(2020, 6, 1)
-    Timecop.freeze(reference_time) { example.run }
-  end
-
     describe '.events_for_category' do
       it 'gets the event names for given category' do
         expect(described_class.events_for_category(:analytics)).to contain_exactly(weekly_event, daily_event)
@@ -154,7 +166,7 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s
       end
 
       it 'raise error if metrics are not in the same category' do
-      expect { described_class.unique_events(event_names: [category_analytics, category_productivity], start_date: 4.weeks.ago, end_date: Date.current) }.to raise_error('Events should be in same category')
+        expect { described_class.unique_events(event_names: [category_analytics_event, category_productivity_event], start_date: 4.weeks.ago, end_date: Date.current) }.to raise_error('Events should be in same category')
       end
 
       it "raise error if metrics don't have same aggregation" do
@@ -183,4 +195,46 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s
         it { expect(described_class.unique_events(event_names: no_slot, start_date: 7.days.ago, end_date: Date.current)).to eq(1) }
       end
     end
+  end
+
+  describe 'unique_events_data' do
+    let(:known_events) do
+      [
+        { name: 'event1_slot', redis_slot: "slot", category: 'category1', aggregation: "weekly" },
+        { name: 'event2_slot', redis_slot: "slot", category: 'category1', aggregation: "weekly" },
+        { name: 'event3', category: 'category2', aggregation: "weekly" },
+        { name: 'event4', category: 'category2', aggregation: "weekly" }
+      ].map(&:with_indifferent_access)
+    end
+
+    before do
+      allow(described_class).to receive(:known_events).and_return(known_events)
+      allow(described_class).to receive(:categories).and_return(%w(category1 category2))
+
+      described_class.track_event(entity1, 'event1_slot', 2.days.ago)
+      described_class.track_event(entity2, 'event2_slot', 2.days.ago)
+      described_class.track_event(entity3, 'event2_slot', 2.weeks.ago)
+
+      # events in different slots
+      described_class.track_event(entity2, 'event3', 2.days.ago)
+      described_class.track_event(entity2, 'event4', 2.days.ago)
+    end
+
+    it 'returns the number of unique events for all known events' do
+      results = {
+       'category1' => {
+          'event1_slot' => 1,
+          'event2_slot' => 1,
+          'category1_total_unique_counts_weekly' => 2,
+          'category1_total_unique_counts_monthly' => 3
+       },
+       'category2' => {
+          'event3' => 1,
+          'event4' => 1
+       }
+      }
+
+      expect(subject.unique_events_data).to eq(results)
+    end
+  end
 end

spec/lib/gitlab/usage_data_spec.rb

@@ -1141,6 +1141,24 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
     end
   end
 
+  describe 'redis_hll_counters' do
+    subject { described_class.redis_hll_counters }
+
+    let(:categories) { ::Gitlab::UsageDataCounters::HLLRedisCounter.categories }
+
+    it 'has all know_events' do
+      expect(subject).to have_key(:redis_hll_counters)
+
+      expect(subject[:redis_hll_counters].keys).to match_array(categories)
+
+      categories.each do |category|
+        keys = ::Gitlab::UsageDataCounters::HLLRedisCounter.events_for_category(category) + ["#{category}_total_unique_counts_weekly", "#{category}_total_unique_counts_monthly"]
+
+        expect(subject[:redis_hll_counters][category].keys).to match_array(keys)
+      end
+    end
+  end
+
   describe '.service_desk_counts' do
     subject { described_class.send(:service_desk_counts) }