Add observability to Snowplow tracking

There is very little visibility into how many events are reported
correctly and how many fail, this makes it hard to investigate any
occurring issues. This commit adds 3 new Promethues metrics that
tracks successfull, failed and total numebr of events, as well as
it logs all failed events.

Changelog: added

doc/administration/monitoring/prometheus/gitlab_metrics.md

@@ -136,6 +136,9 @@ The following metrics are available:
 | `service_desk_thank_you_email`                                   | Counter     | 14.0    | Total number of email responses to new service desk emails                                                            |                                                           |
 | `service_desk_new_note_email`                                    | Counter     | 14.0    | Total number of email notifications on new service desk comment                                                       |                                                           |
 | `email_receiver_error`                                           | Counter     | 14.1    | Total number of errors when processing incoming emails                                                                |                                                           |
+| `gitlab_snowplow_events_total`                                   | Counter     | 14.1    | Total number of GitLab Snowplow product intelligence events emitted                                                   |                                                           |
+| `gitlab_snowplow_failed_events_total`                            | Counter     | 14.1    | Total number of GitLab Snowplow product intelligence events emission failures                                         |                                                           |
+| `gitlab_snowplow_successful_events_total`                        | Counter     | 14.1    | Total number of GitLab Snowplow product intelligence events emission successes                                        |                                                           |
 
 ## Metrics controlled by a feature flag
 

lib/gitlab/tracking/destinations/snowplow.rb

@@ -13,6 +13,7 @@ module Gitlab
           return unless enabled?
 
           tracker.track_struct_event(category, action, label, property, value, context, (Time.now.to_f * 1000).to_i)
+          increment_total_events_counter
         end
 
         private
@@ -33,9 +34,46 @@ module Gitlab
         def emitter
           SnowplowTracker::AsyncEmitter.new(
             Gitlab::CurrentSettings.snowplow_collector_hostname,
-            protocol: 'https'
+            protocol: 'https',
+            on_success: method(:increment_successful_events_emissions),
+            on_failure: method(:failure_callback)
           )
         end
+
+        def failure_callback(success_count, failures)
+          increment_successful_events_emissions(success_count)
+          increment_failed_events_emissions(failures.size)
+          log_failures(failures)
+        end
+
+        def increment_failed_events_emissions(value)
+          Gitlab::Metrics.counter(
+            :gitlab_snowplow_failed_events_total,
+            'Number of failed Snowplow events emissions'
+          ).increment({}, value.to_i)
+        end
+
+        def increment_successful_events_emissions(value)
+          Gitlab::Metrics.counter(
+            :gitlab_snowplow_successful_events_total,
+            'Number of successful Snowplow events emissions'
+          ).increment({}, value.to_i)
+        end
+
+        def increment_total_events_counter
+          Gitlab::Metrics.counter(
+            :gitlab_snowplow_events_total,
+            'Number of Snowplow events'
+          ).increment
+        end
+
+        def log_failures(failures)
+          hostname = Gitlab::CurrentSettings.snowplow_collector_hostname
+
+          failures.each do |failure|
+            Gitlab::AppLogger.error("#{failure["se_ca"]} #{failure["se_ac"]} failed to be reported to collector at #{hostname}")
+          end
+        end
       end
     end
   end

spec/lib/gitlab/tracking/destinations/snowplow_spec.rb

@@ -21,7 +21,10 @@ RSpec.describe Gitlab::Tracking::Destinations::Snowplow do
 
       expect(SnowplowTracker::AsyncEmitter)
         .to receive(:new)
-        .with('gitfoo.com', { protocol: 'https' })
+        .with('gitfoo.com',
+              { protocol: 'https',
+                on_success: subject.method(:increment_successful_events_emissions),
+                on_failure: subject.method(:failure_callback) })
         .and_return(emitter)
 
       expect(SnowplowTracker::Tracker)
@@ -40,6 +43,18 @@ RSpec.describe Gitlab::Tracking::Destinations::Snowplow do
           .to have_received(:track_struct_event)
           .with('category', 'action', 'label', 'property', 1.5, nil, (Time.now.to_f * 1000).to_i)
       end
+
+      it 'increase total snowplow events counter' do
+        counter = double
+
+        expect(counter).to receive(:increment)
+        expect(Gitlab::Metrics).to receive(:counter)
+                                     .with(:gitlab_snowplow_events_total,
+                                           'Number of Snowplow events')
+                                     .and_return(counter)
+
+        subject.event('category', 'action', label: 'label', property: 'property', value: 1.5)
+      end
     end
   end
 
@@ -52,4 +67,43 @@ RSpec.describe Gitlab::Tracking::Destinations::Snowplow do
       end
     end
   end
+
+  context 'callbacks' do
+    describe 'on success' do
+      it 'increase gitlab_successful_snowplow_events_total counter' do
+        counter = double
+
+        expect(counter).to receive(:increment).with({}, 2)
+        expect(Gitlab::Metrics).to receive(:counter)
+                                     .with(:gitlab_snowplow_successful_events_total,
+                                           'Number of successful Snowplow events emissions')
+                                     .and_return(counter)
+
+        subject.method(:increment_successful_events_emissions).call(2)
+      end
+    end
+
+    describe 'on failure' do
+      it 'increase gitlab_failed_snowplow_events_total counter and logs failures', :aggregate_failures do
+        counter = double
+        error_message = "Admin::AuditLogsController search_audit_event failed to be reported to collector at gitfoo.com"
+        failures = [{ "e" => "se",
+                      "se_ca" => "Admin::AuditLogsController",
+                      "se_ac" => "search_audit_event" }]
+        allow(Gitlab::Metrics).to receive(:counter)
+                                    .with(:gitlab_snowplow_successful_events_total,
+                                          'Number of successful Snowplow events emissions')
+                                    .and_call_original
+
+        expect(Gitlab::AppLogger).to receive(:error).with(error_message)
+        expect(counter).to receive(:increment).with({}, 1)
+        expect(Gitlab::Metrics).to receive(:counter)
+                                     .with(:gitlab_snowplow_failed_events_total,
+                                           'Number of failed Snowplow events emissions')
+                                     .and_return(counter)
+
+        subject.method(:failure_callback).call(2, failures)
+      end
+    end
+  end
 end