Transaction metrics from workers are not collected

Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/342578#note_699445484
Changelog: fixed

config/initializers/zz_metrics.rb

@@ -42,7 +42,7 @@ if Gitlab::Metrics.enabled? && !Rails.env.test? && !(Rails.env.development? && d
     def connect(*args)
       val = super
 
-      if current_transaction = ::Gitlab::Metrics::Transaction.current
+      if current_transaction = (::Gitlab::Metrics::WebTransaction.current || ::Gitlab::Metrics::BackgroundTransaction.current)
         current_transaction.increment(:gitlab_transaction_new_redis_connections_total, 1)
       end
 

lib/gitlab/diff/highlight_cache.rb

@@ -213,7 +213,7 @@ module Gitlab
       end
 
       def current_transaction
-        ::Gitlab::Metrics::Transaction.current
+        ::Gitlab::Metrics::WebTransaction.current
       end
     end
   end

lib/gitlab/metrics.rb

@@ -29,7 +29,7 @@ module Gitlab
 
     # Allow access from other metrics related middlewares
     def self.current_transaction
-      Transaction.current
+      WebTransaction.current || BackgroundTransaction.current
     end
 
     # Returns the prefix to use for the name of a series.

lib/gitlab/metrics/background_transaction.rb

@@ -2,14 +2,17 @@
 
 module Gitlab
   module Metrics
+    # Exclusive transaction-type metrics for background jobs (Sidekiq). One
+    # instance of this class is created for each job going through the Sidekiq
+    # metric middleware. Any metrics dispatched with this instance include
+    # metadata such as endpoint_id, queue, and feature category.
     class BackgroundTransaction < Transaction
-      # Separate web transaction instance and background transaction instance
-      BACKGROUND_THREAD_KEY = :_gitlab_metrics_background_transaction
-      BACKGROUND_BASE_LABEL_KEYS = %i(endpoint_id feature_category).freeze
+      THREAD_KEY = :_gitlab_metrics_background_transaction
+      BASE_LABEL_KEYS = %i(queue endpoint_id feature_category).freeze
 
       class << self
         def current
-          Thread.current[BACKGROUND_THREAD_KEY]
+          Thread.current[THREAD_KEY]
         end
 
         def prometheus_metric(name, type, &block)
@@ -19,17 +22,17 @@ module Gitlab
 
             evaluate(&block)
             # always filter sensitive labels and merge with base ones
-            label_keys BACKGROUND_BASE_LABEL_KEYS | (label_keys - ::Gitlab::Metrics::Transaction::FILTERED_LABEL_KEYS)
+            label_keys BASE_LABEL_KEYS | (label_keys - ::Gitlab::Metrics::Transaction::FILTERED_LABEL_KEYS)
           end
         end
       end
 
       def run
-        Thread.current[BACKGROUND_THREAD_KEY] = self
+        Thread.current[THREAD_KEY] = self
 
         yield
       ensure
-        Thread.current[BACKGROUND_THREAD_KEY] = nil
+        Thread.current[THREAD_KEY] = nil
       end
 
       def labels

lib/gitlab/metrics/methods.rb

@@ -13,8 +13,12 @@ module Gitlab
       end
 
       class_methods do
-        def reload_metric!(name)
-          @@_metrics_provider_cache.delete(name)
+        def reload_metric!(name = nil)
+          if name.nil?
+            @@_metrics_provider_cache = {}
+          else
+            @@_metrics_provider_cache.delete(name)
+          end
         end
 
         private

lib/gitlab/metrics/subscribers/action_view.rb

@@ -40,7 +40,7 @@ module Gitlab
         end
 
         def current_transaction
-          ::Gitlab::Metrics::Transaction.current
+          ::Gitlab::Metrics::WebTransaction.current
         end
       end
     end

lib/gitlab/metrics/subscribers/external_http.rb

@@ -43,7 +43,7 @@ module Gitlab
         private
 
         def current_transaction
-          ::Gitlab::Metrics::Transaction.current
+          ::Gitlab::Metrics::WebTransaction.current || ::Gitlab::Metrics::BackgroundTransaction.current
         end
 
         def add_to_detail_store(start, payload)

lib/gitlab/metrics/subscribers/rails_cache.rb

@@ -65,7 +65,7 @@ module Gitlab
         private
 
         def current_transaction
-          ::Gitlab::Metrics::Transaction.current
+          ::Gitlab::Metrics::WebTransaction.current
         end
 
         def metric_cache_operation_duration_seconds

lib/gitlab/metrics/transaction.rb

@@ -6,35 +6,14 @@ module Gitlab
     class Transaction
       include Gitlab::Metrics::Methods
 
-      # base label keys shared among all transactions
-      BASE_LABEL_KEYS = %i(controller action feature_category).freeze
       # labels that potentially contain sensitive information and will be filtered
       FILTERED_LABEL_KEYS = %i(branch path).freeze
 
-      THREAD_KEY = :_gitlab_metrics_transaction
-
       # The series to store events (e.g. Git pushes) in.
       EVENT_SERIES = 'events'
 
       attr_reader :method
 
-      class << self
-        def current
-          Thread.current[THREAD_KEY]
-        end
-
-        def prometheus_metric(name, type, &block)
-          fetch_metric(type, name) do
-            # set default metric options
-            docstring "#{name.to_s.humanize} #{type}"
-
-            evaluate(&block)
-            # always filter sensitive labels and merge with base ones
-            label_keys BASE_LABEL_KEYS | (label_keys - FILTERED_LABEL_KEYS)
-          end
-        end
-      end
-
       def initialize
         @methods = {}
       end
@@ -126,10 +105,6 @@ module Gitlab
         histogram.observe(filter_labels(labels), value)
       end
 
-      def labels
-        BASE_LABEL_KEYS.product([nil]).to_h
-      end
-
       def filter_labels(labels)
         labels.empty? ? self.labels : labels.without(*FILTERED_LABEL_KEYS).merge(self.labels)
       end

lib/gitlab/metrics/web_transaction.rb

@@ -2,12 +2,37 @@
 
 module Gitlab
   module Metrics
+    # Exclusive transaction-type metrics for web servers (including Web/Api/Git
+    # fleet). One instance of this class is created for each request going
+    # through the Rack metric middleware. Any metrics dispatched with this
+    # instance include metadata such as controller, action, feature category,
+    # etc.
     class WebTransaction < Transaction
+      THREAD_KEY = :_gitlab_metrics_transaction
+      BASE_LABEL_KEYS = %i(controller action feature_category).freeze
+
       CONTROLLER_KEY = 'action_controller.instance'
       ENDPOINT_KEY = 'api.endpoint'
       ALLOWED_SUFFIXES = Set.new(%w[json js atom rss xml zip])
       SMALL_BUCKETS = [0.1, 0.25, 0.5, 1.0, 2.5, 5.0].freeze
 
+      class << self
+        def current
+          Thread.current[THREAD_KEY]
+        end
+
+        def prometheus_metric(name, type, &block)
+          fetch_metric(type, name) do
+            # set default metric options
+            docstring "#{name.to_s.humanize} #{type}"
+
+            evaluate(&block)
+            # always filter sensitive labels and merge with base ones
+            label_keys BASE_LABEL_KEYS | (label_keys - ::Gitlab::Metrics::Transaction::FILTERED_LABEL_KEYS)
+          end
+        end
+      end
+
       def initialize(env)
         super()
         @env = env

spec/lib/gitlab/metrics/background_transaction_spec.rb

@@ -4,27 +4,28 @@ require 'spec_helper'
 
 RSpec.describe Gitlab::Metrics::BackgroundTransaction do
   let(:transaction) { described_class.new }
-  let(:prometheus_metric) { instance_double(Prometheus::Client::Metric, base_labels: {}) }
-
-  before do
-    allow(described_class).to receive(:prometheus_metric).and_return(prometheus_metric)
-  end
 
   describe '#run' do
+    let(:prometheus_metric) { instance_double(Prometheus::Client::Metric, base_labels: {}) }
+
+    before do
+      allow(described_class).to receive(:prometheus_metric).and_return(prometheus_metric)
+    end
+
     it 'yields the supplied block' do
       expect { |b| transaction.run(&b) }.to yield_control
     end
 
     it 'stores the transaction in the current thread' do
       transaction.run do
-        expect(Thread.current[described_class::BACKGROUND_THREAD_KEY]).to eq(transaction)
+        expect(Thread.current[described_class::THREAD_KEY]).to eq(transaction)
       end
     end
 
     it 'removes the transaction from the current thread upon completion' do
       transaction.run { }
 
-      expect(Thread.current[described_class::BACKGROUND_THREAD_KEY]).to be_nil
+      expect(Thread.current[described_class::THREAD_KEY]).to be_nil
     end
   end
 
@@ -68,7 +69,10 @@ RSpec.describe Gitlab::Metrics::BackgroundTransaction do
     end
   end
 
-  RSpec.shared_examples 'metric with labels' do |metric_method|
+  it_behaves_like 'transaction metrics with labels' do
+    let(:transaction_obj) { described_class.new }
+    let(:labels) { { endpoint_id: 'TestWorker', feature_category: 'projects', queue: 'test_worker' } }
+
     before do
       test_worker_class = Class.new do
         def self.queue
@@ -78,33 +82,10 @@ RSpec.describe Gitlab::Metrics::BackgroundTransaction do
       stub_const('TestWorker', test_worker_class)
     end
 
-    it 'measures with correct labels and value' do
-      value = 1
-      expect(prometheus_metric).to receive(metric_method).with({
-        endpoint_id: 'TestWorker', feature_category: 'projects', queue: 'test_worker'
-      }, value)
-
+    around do |example|
       Gitlab::ApplicationContext.with_raw_context(feature_category: 'projects', caller_id: 'TestWorker') do
-        transaction.send(metric_method, :test_metric, value)
+        example.run
       end
     end
   end
-
-  describe '#increment' do
-    let(:prometheus_metric) { instance_double(Prometheus::Client::Counter, :increment, base_labels: {}) }
-
-    it_behaves_like 'metric with labels', :increment
-  end
-
-  describe '#set' do
-    let(:prometheus_metric) { instance_double(Prometheus::Client::Gauge, :set, base_labels: {}) }
-
-    it_behaves_like 'metric with labels', :set
-  end
-
-  describe '#observe' do
-    let(:prometheus_metric) { instance_double(Prometheus::Client::Histogram, :observe, base_labels: {}) }
-
-    it_behaves_like 'metric with labels', :observe
-  end
 end

spec/lib/gitlab/metrics/method_call_spec.rb

@@ -37,7 +37,7 @@ RSpec.describe Gitlab::Metrics::MethodCall do
 
         it 'metric is not a NullMetric' do
           method_call.measure { 'foo' }
-          expect(::Gitlab::Metrics::Transaction.prometheus_metric(:gitlab_method_call_duration_seconds, :histogram)).not_to be_instance_of(Gitlab::Metrics::NullMetric)
+          expect(::Gitlab::Metrics::WebTransaction.prometheus_metric(:gitlab_method_call_duration_seconds, :histogram)).not_to be_instance_of(Gitlab::Metrics::NullMetric)
         end
 
         it 'observes the performance of the supplied block' do
@@ -63,7 +63,7 @@ RSpec.describe Gitlab::Metrics::MethodCall do
         it 'observes using NullMetric' do
           method_call.measure { 'foo' }
 
-          expect(::Gitlab::Metrics::Transaction.prometheus_metric(:gitlab_method_call_duration_seconds, :histogram)).to be_instance_of(Gitlab::Metrics::NullMetric)
+          expect(::Gitlab::Metrics::WebTransaction.prometheus_metric(:gitlab_method_call_duration_seconds, :histogram)).to be_instance_of(Gitlab::Metrics::NullMetric)
         end
       end
     end

spec/lib/gitlab/metrics/subscribers/external_http_spec.rb

@@ -3,7 +3,7 @@
 require 'spec_helper'
 
 RSpec.describe Gitlab::Metrics::Subscribers::ExternalHttp, :request_store do
-  let(:transaction) { Gitlab::Metrics::Transaction.new }
+  let(:transaction) { Gitlab::Metrics::WebTransaction.new({}) }
   let(:subscriber) { described_class.new }
 
   around do |example|

spec/lib/gitlab/metrics/transaction_spec.rb

@@ -3,172 +3,7 @@
 require 'spec_helper'
 
 RSpec.describe Gitlab::Metrics::Transaction do
-  let(:transaction) { described_class.new }
-
-  let(:sensitive_tags) do
-    {
-      path: 'private',
-      branch: 'sensitive'
-    }
-  end
-
-  describe '#method_call_for' do
-    it 'returns a MethodCall' do
-      method = transaction.method_call_for('Foo#bar', :Foo, '#bar')
-
-      expect(method).to be_an_instance_of(Gitlab::Metrics::MethodCall)
-    end
-  end
-
   describe '#run' do
-    specify { expect { transaction.run }.to raise_error(NotImplementedError) }
-  end
-
-  describe '#add_event' do
-    let(:prometheus_metric) { instance_double(Prometheus::Client::Counter, increment: nil, base_labels: {}) }
-
-    it 'adds a metric' do
-      expect(prometheus_metric).to receive(:increment)
-      expect(described_class).to receive(:fetch_metric).with(:counter, :gitlab_transaction_event_meow_total).and_return(prometheus_metric)
-
-      transaction.add_event(:meow)
-    end
-
-    it 'allows tracking of custom tags' do
-      expect(prometheus_metric).to receive(:increment).with(hash_including(animal: "dog"))
-      expect(described_class).to receive(:fetch_metric).with(:counter, :gitlab_transaction_event_bau_total).and_return(prometheus_metric)
-
-      transaction.add_event(:bau, animal: 'dog')
-    end
-
-    context 'with sensitive tags' do
-      before do
-        transaction.add_event(:baubau, **sensitive_tags.merge(sane: 'yes'))
-        allow(described_class).to receive(:prometheus_metric).and_return(prometheus_metric)
-      end
-
-      it 'filters tags' do
-        expect(prometheus_metric).not_to receive(:increment).with(hash_including(sensitive_tags))
-
-        transaction.add_event(:baubau, **sensitive_tags.merge(sane: 'yes'))
-      end
-    end
-  end
-
-  describe '#increment' do
-    let(:prometheus_metric) { instance_double(Prometheus::Client::Counter, increment: nil, base_labels: {}) }
-
-    it 'adds a metric' do
-      expect(prometheus_metric).to receive(:increment)
-      expect(::Gitlab::Metrics).to receive(:counter).with(:meow, 'Meow counter', hash_including(:controller, :action)).and_return(prometheus_metric)
-
-      transaction.increment(:meow, 1)
-    end
-
-    context 'with block' do
-      it 'overrides docstring' do
-        expect(::Gitlab::Metrics).to receive(:counter).with(:block_docstring, 'test', hash_including(:controller, :action)).and_return(prometheus_metric)
-
-        transaction.increment(:block_docstring, 1) do
-          docstring 'test'
-        end
-      end
-
-      it 'overrides labels' do
-        expect(::Gitlab::Metrics).to receive(:counter).with(:block_labels, 'Block labels counter', hash_including(:controller, :action, :sane)).and_return(prometheus_metric)
-
-        labels = { sane: 'yes' }
-        transaction.increment(:block_labels, 1, labels) do
-          label_keys %i(sane)
-        end
-      end
-
-      it 'filters sensitive tags' do
-        expect(::Gitlab::Metrics).to receive(:counter).with(:metric_with_sensitive_block, 'Metric with sensitive block counter', hash_excluding(sensitive_tags)).and_return(prometheus_metric)
-
-        labels_keys = sensitive_tags.keys
-        transaction.increment(:metric_with_sensitive_block, 1, sensitive_tags) do
-          label_keys labels_keys
-        end
-      end
-    end
-  end
-
-  describe '#set' do
-    let(:prometheus_metric) { instance_double(Prometheus::Client::Gauge, set: nil, base_labels: {}) }
-
-    it 'adds a metric' do
-      expect(prometheus_metric).to receive(:set)
-      expect(::Gitlab::Metrics).to receive(:gauge).with(:meow_set, 'Meow set gauge', hash_including(:controller, :action), :all).and_return(prometheus_metric)
-
-      transaction.set(:meow_set, 1)
-    end
-
-    context 'with block' do
-      it 'overrides docstring' do
-        expect(::Gitlab::Metrics).to receive(:gauge).with(:block_docstring_set, 'test', hash_including(:controller, :action), :all).and_return(prometheus_metric)
-
-        transaction.set(:block_docstring_set, 1) do
-          docstring 'test'
-        end
-      end
-
-      it 'overrides labels' do
-        expect(::Gitlab::Metrics).to receive(:gauge).with(:block_labels_set, 'Block labels set gauge', hash_including(:controller, :action, :sane), :all).and_return(prometheus_metric)
-
-        labels = { sane: 'yes' }
-        transaction.set(:block_labels_set, 1, labels) do
-          label_keys %i(sane)
-        end
-      end
-
-      it 'filters sensitive tags' do
-        expect(::Gitlab::Metrics).to receive(:gauge).with(:metric_set_with_sensitive_block, 'Metric set with sensitive block gauge', hash_excluding(sensitive_tags), :all).and_return(prometheus_metric)
-
-        label_keys = sensitive_tags.keys
-        transaction.set(:metric_set_with_sensitive_block, 1, sensitive_tags) do
-          label_keys label_keys
-        end
-      end
-    end
-  end
-
-  describe '#observe' do
-    let(:prometheus_metric) { instance_double(Prometheus::Client::Histogram, observe: nil, base_labels: {}) }
-
-    it 'adds a metric' do
-      expect(prometheus_metric).to receive(:observe)
-      expect(::Gitlab::Metrics).to receive(:histogram).with(:meow_observe, 'Meow observe histogram', hash_including(:controller, :action), kind_of(Array)).and_return(prometheus_metric)
-
-      transaction.observe(:meow_observe, 1)
-    end
-
-    context 'with block' do
-      it 'overrides docstring' do
-        expect(::Gitlab::Metrics).to receive(:histogram).with(:block_docstring_observe, 'test', hash_including(:controller, :action), kind_of(Array)).and_return(prometheus_metric)
-
-        transaction.observe(:block_docstring_observe, 1) do
-          docstring 'test'
-        end
-      end
-
-      it 'overrides labels' do
-        expect(::Gitlab::Metrics).to receive(:histogram).with(:block_labels_observe, 'Block labels observe histogram', hash_including(:controller, :action, :sane), kind_of(Array)).and_return(prometheus_metric)
-
-        labels = { sane: 'yes' }
-        transaction.observe(:block_labels_observe, 1, labels) do
-          label_keys %i(sane)
-        end
-      end
-
-      it 'filters sensitive tags' do
-        expect(::Gitlab::Metrics).to receive(:histogram).with(:metric_observe_with_sensitive_block, 'Metric observe with sensitive block histogram', hash_excluding(sensitive_tags), kind_of(Array)).and_return(prometheus_metric)
-
-        label_keys = sensitive_tags.keys
-        transaction.observe(:metric_observe_with_sensitive_block, 1, sensitive_tags) do
-          label_keys label_keys
-        end
-      end
-    end
+    specify { expect { described_class.new.run }.to raise_error(NotImplementedError) }
   end
 end

spec/lib/gitlab/metrics/web_transaction_spec.rb

@@ -5,41 +5,14 @@ require 'spec_helper'
 RSpec.describe Gitlab::Metrics::WebTransaction do
   let(:env) { {} }
   let(:transaction) { described_class.new(env) }
-  let(:prometheus_metric) { instance_double(Prometheus::Client::Metric, base_labels: {}) }
 
-  before do
-    allow(described_class).to receive(:prometheus_metric).and_return(prometheus_metric)
-  end
-
-  RSpec.shared_context 'ActionController request' do
-    let(:request) { double(:request, format: double(:format, ref: :html)) }
-    let(:controller_class) { double(:controller_class, name: 'TestController') }
-
-    before do
-      controller = double(:controller, class: controller_class, action_name: 'show', request: request)
-      env['action_controller.instance'] = controller
-    end
-  end
+  describe '#run' do
+    let(:prometheus_metric) { instance_double(Prometheus::Client::Metric, base_labels: {}) }
 
-  RSpec.shared_context 'transaction observe metrics' do
     before do
+      allow(described_class).to receive(:prometheus_metric).and_return(prometheus_metric)
       allow(transaction).to receive(:observe)
     end
-  end
-
-  RSpec.shared_examples 'metric with labels' do |metric_method|
-    include_context 'ActionController request'
-
-    it 'measures with correct labels and value' do
-      value = 1
-      expect(prometheus_metric).to receive(metric_method).with({ controller: 'TestController', action: 'show', feature_category: ::Gitlab::FeatureCategories::FEATURE_CATEGORY_DEFAULT }, value)
-
-      transaction.send(metric_method, :bau, value)
-    end
-  end
-
-  describe '#run' do
-    include_context 'transaction observe metrics'
 
     it 'yields the supplied block' do
       expect { |b| transaction.run(&b) }.to yield_control
@@ -88,14 +61,6 @@ RSpec.describe Gitlab::Metrics::WebTransaction do
     end
   end
 
-  describe '#method_call_for' do
-    it 'returns a MethodCall' do
-      method = transaction.method_call_for('Foo#bar', :Foo, '#bar')
-
-      expect(method).to be_an_instance_of(Gitlab::Metrics::MethodCall)
-    end
-  end
-
   describe '#labels' do
     context 'when request goes to Grape endpoint' do
       before do
@@ -115,7 +80,7 @@ RSpec.describe Gitlab::Metrics::WebTransaction do
       end
 
       it 'contains only the labels defined for transactions' do
-        expect(transaction.labels.keys).to contain_exactly(*described_class.superclass::BASE_LABEL_KEYS)
+        expect(transaction.labels.keys).to contain_exactly(*described_class::BASE_LABEL_KEYS)
       end
 
       it 'does not provide labels if route infos are missing' do
@@ -129,14 +94,20 @@ RSpec.describe Gitlab::Metrics::WebTransaction do
     end
 
     context 'when request goes to ActionController' do
-      include_context 'ActionController request'
+      let(:request) { double(:request, format: double(:format, ref: :html)) }
+      let(:controller_class) { double(:controller_class, name: 'TestController') }
+
+      before do
+        controller = double(:controller, class: controller_class, action_name: 'show', request: request)
+        env['action_controller.instance'] = controller
+      end
 
       it 'tags a transaction with the name and action of a controller' do
         expect(transaction.labels).to eq({ controller: 'TestController', action: 'show', feature_category: ::Gitlab::FeatureCategories::FEATURE_CATEGORY_DEFAULT })
       end
 
       it 'contains only the labels defined for transactions' do
-        expect(transaction.labels.keys).to contain_exactly(*described_class.superclass::BASE_LABEL_KEYS)
+        expect(transaction.labels.keys).to contain_exactly(*described_class::BASE_LABEL_KEYS)
       end
 
       context 'when the request content type is not :html' do
@@ -170,37 +141,16 @@ RSpec.describe Gitlab::Metrics::WebTransaction do
     end
   end
 
-  describe '#add_event' do
-    let(:prometheus_metric) { instance_double(Prometheus::Client::Counter, :increment, base_labels: {}) }
-
-    it 'adds a metric' do
-      expect(prometheus_metric).to receive(:increment)
-
-      transaction.add_event(:meow)
-    end
+  it_behaves_like 'transaction metrics with labels' do
+    let(:request) { double(:request, format: double(:format, ref: :html)) }
+    let(:controller_class) { double(:controller_class, name: 'TestController') }
+    let(:controller) { double(:controller, class: controller_class, action_name: 'show', request: request) }
 
-    it 'allows tracking of custom tags' do
-      expect(prometheus_metric).to receive(:increment).with(animal: "dog")
+    let(:transaction_obj) { described_class.new({ 'action_controller.instance' => controller }) }
+    let(:labels) { { controller: 'TestController', action: 'show', feature_category: 'projects' } }
 
-      transaction.add_event(:bau, animal: 'dog')
+    before do
+      ::Gitlab::ApplicationContext.push(feature_category: 'projects')
     end
   end
-
-  describe '#increment' do
-    let(:prometheus_metric) { instance_double(Prometheus::Client::Counter, :increment, base_labels: {}) }
-
-    it_behaves_like 'metric with labels', :increment
-  end
-
-  describe '#set' do
-    let(:prometheus_metric) { instance_double(Prometheus::Client::Gauge, :set, base_labels: {}) }
-
-    it_behaves_like 'metric with labels', :set
-  end
-
-  describe '#observe' do
-    let(:prometheus_metric) { instance_double(Prometheus::Client::Histogram, :observe, base_labels: {}) }
-
-    it_behaves_like 'metric with labels', :observe
-  end
 end

spec/support/shared_examples/metrics/transaction_metrics_with_labels_shared_examples.rb

+# frozen_string_literal: true
+
+RSpec.shared_examples 'transaction metrics with labels' do
+  let(:sensitive_tags) do
+    {
+      path: 'private',
+      branch: 'sensitive'
+    }
+  end
+
+  around do |example|
+    described_class.reload_metric!
+    example.run
+    described_class.reload_metric!
+  end
+
+  describe '.prometheus_metric' do
+    let(:prometheus_metric) { instance_double(Prometheus::Client::Histogram, observe: nil, base_labels: {}) }
+
+    it 'adds a metric' do
+      expect(::Gitlab::Metrics).to receive(:histogram).with(
+        :meow_observe, 'Meow observe histogram', hash_including(*described_class::BASE_LABEL_KEYS), be_a(Array)
+      ).and_return(prometheus_metric)
+
+      expect do |block|
+        metric = described_class.prometheus_metric(:meow_observe, :histogram, &block)
+        expect(metric).to be(prometheus_metric)
+      end.to yield_control
+    end
+  end
+
+  describe '#method_call_for' do
+    it 'returns a MethodCall' do
+      method = transaction_obj.method_call_for('Foo#bar', :Foo, '#bar')
+
+      expect(method).to be_an_instance_of(Gitlab::Metrics::MethodCall)
+    end
+  end
+
+  describe '#add_event' do
+    let(:prometheus_metric) { instance_double(Prometheus::Client::Counter, increment: nil, base_labels: {}) }
+
+    it 'adds a metric' do
+      expect(prometheus_metric).to receive(:increment).with(labels)
+      expect(described_class).to receive(:fetch_metric).with(:counter, :gitlab_transaction_event_meow_total).and_return(prometheus_metric)
+
+      transaction_obj.add_event(:meow)
+    end
+
+    it 'allows tracking of custom tags' do
+      expect(prometheus_metric).to receive(:increment).with(labels.merge(animal: "dog"))
+      expect(described_class).to receive(:fetch_metric).with(:counter, :gitlab_transaction_event_bau_total).and_return(prometheus_metric)
+
+      transaction_obj.add_event(:bau, animal: 'dog')
+    end
+
+    context 'with sensitive tags' do
+      it 'filters tags' do
+        expect(described_class).to receive(:fetch_metric).with(:counter, :gitlab_transaction_event_bau_total).and_return(prometheus_metric)
+        expect(prometheus_metric).not_to receive(:increment).with(hash_including(sensitive_tags))
+
+        transaction_obj.add_event(:bau, **sensitive_tags.merge(sane: 'yes'))
+      end
+    end
+  end
+
+  describe '#increment' do
+    let(:prometheus_metric) { instance_double(Prometheus::Client::Counter, increment: nil, base_labels: {}) }
+
+    it 'adds a metric' do
+      expect(::Gitlab::Metrics).to receive(:counter).with(
+        :meow, 'Meow counter', hash_including(*described_class::BASE_LABEL_KEYS)
+      ).and_return(prometheus_metric)
+      expect(prometheus_metric).to receive(:increment).with(labels, 1)
+
+      transaction_obj.increment(:meow, 1)
+    end
+
+    context 'with block' do
+      it 'overrides docstring' do
+        expect(::Gitlab::Metrics).to receive(:counter).with(
+          :block_docstring, 'test', hash_including(*described_class::BASE_LABEL_KEYS)
+        ).and_return(prometheus_metric)
+        expect(prometheus_metric).to receive(:increment).with(labels, 1)
+
+        transaction_obj.increment(:block_docstring, 1) do
+          docstring 'test'
+        end
+      end
+
+      it 'overrides labels' do
+        expect(::Gitlab::Metrics).to receive(:counter).with(
+          :block_labels, 'Block labels counter', hash_including(*described_class::BASE_LABEL_KEYS)
+        ).and_return(prometheus_metric)
+        expect(prometheus_metric).to receive(:increment).with(labels.merge(sane: 'yes'), 1)
+
+        transaction_obj.increment(:block_labels, 1, sane: 'yes') do
+          label_keys %i(sane)
+        end
+      end
+
+      it 'filters sensitive tags' do
+        labels_keys = sensitive_tags.keys
+
+        expect(::Gitlab::Metrics).to receive(:counter).with(
+          :metric_with_sensitive_block, 'Metric with sensitive block counter', hash_excluding(labels_keys)
+        ).and_return(prometheus_metric)
+        expect(prometheus_metric).to receive(:increment).with(labels, 1)
+
+        transaction_obj.increment(:metric_with_sensitive_block, 1, sensitive_tags) do
+          label_keys labels_keys
+        end
+      end
+    end
+  end
+
+  describe '#set' do
+    let(:prometheus_metric) { instance_double(Prometheus::Client::Gauge, set: nil, base_labels: {}) }
+
+    it 'adds a metric' do
+      expect(::Gitlab::Metrics).to receive(:gauge).with(
+        :meow_set, 'Meow set gauge', hash_including(*described_class::BASE_LABEL_KEYS), :all
+      ).and_return(prometheus_metric)
+      expect(prometheus_metric).to receive(:set).with(labels, 99)
+
+      transaction_obj.set(:meow_set, 99)
+    end
+
+    context 'with block' do
+      it 'overrides docstring' do
+        expect(::Gitlab::Metrics).to receive(:gauge).with(
+          :block_docstring_set, 'test', hash_including(*described_class::BASE_LABEL_KEYS), :all
+        ).and_return(prometheus_metric)
+        expect(prometheus_metric).to receive(:set).with(labels, 99)
+
+        transaction_obj.set(:block_docstring_set, 99) do
+          docstring 'test'
+        end
+      end
+
+      it 'overrides labels' do
+        expect(::Gitlab::Metrics).to receive(:gauge).with(
+          :block_labels_set, 'Block labels set gauge', hash_including(*described_class::BASE_LABEL_KEYS), :all
+        ).and_return(prometheus_metric)
+        expect(prometheus_metric).to receive(:set).with(labels.merge(sane: 'yes'), 99)
+
+        transaction_obj.set(:block_labels_set, 99, sane: 'yes') do
+          label_keys %i(sane)
+        end
+      end
+
+      it 'filters sensitive tags' do
+        labels_keys = sensitive_tags.keys
+
+        expect(::Gitlab::Metrics).to receive(:gauge).with(
+          :metric_set_with_sensitive_block, 'Metric set with sensitive block gauge', hash_excluding(*labels_keys), :all
+        ).and_return(prometheus_metric)
+        expect(prometheus_metric).to receive(:set).with(labels, 99)
+
+        transaction_obj.set(:metric_set_with_sensitive_block, 99, sensitive_tags) do
+          label_keys label_keys
+        end
+      end
+    end
+  end
+
+  describe '#observe' do
+    let(:prometheus_metric) { instance_double(Prometheus::Client::Histogram, observe: nil, base_labels: {}) }
+
+    it 'adds a metric' do
+      expect(::Gitlab::Metrics).to receive(:histogram).with(
+        :meow_observe, 'Meow observe histogram', hash_including(*described_class::BASE_LABEL_KEYS), kind_of(Array)
+      ).and_return(prometheus_metric)
+      expect(prometheus_metric).to receive(:observe).with(labels, 2.0)
+
+      transaction_obj.observe(:meow_observe, 2.0)
+    end
+
+    context 'with block' do
+      it 'overrides docstring' do
+        expect(::Gitlab::Metrics).to receive(:histogram).with(
+          :block_docstring_observe, 'test', hash_including(*described_class::BASE_LABEL_KEYS), kind_of(Array)
+        ).and_return(prometheus_metric)
+        expect(prometheus_metric).to receive(:observe).with(labels, 2.0)
+
+        transaction_obj.observe(:block_docstring_observe, 2.0) do
+          docstring 'test'
+        end
+      end
+
+      it 'overrides labels' do
+        expect(::Gitlab::Metrics).to receive(:histogram).with(
+          :block_labels_observe, 'Block labels observe histogram', hash_including(*described_class::BASE_LABEL_KEYS), kind_of(Array)
+        ).and_return(prometheus_metric)
+        expect(prometheus_metric).to receive(:observe).with(labels.merge(sane: 'yes'), 2.0)
+
+        transaction_obj.observe(:block_labels_observe, 2.0, sane: 'yes') do
+          label_keys %i(sane)
+        end
+      end
+
+      it 'filters sensitive tags' do
+        labels_keys = sensitive_tags.keys
+
+        expect(::Gitlab::Metrics).to receive(:histogram).with(
+          :metric_observe_with_sensitive_block,
+          'Metric observe with sensitive block histogram',
+          hash_excluding(labels_keys),
+          kind_of(Array)
+        ).and_return(prometheus_metric)
+        expect(prometheus_metric).to receive(:observe).with(labels, 2.0)
+
+        transaction_obj.observe(:metric_observe_with_sensitive_block, 2.0, sensitive_tags) do
+          label_keys label_keys
+        end
+      end
+    end
+  end
+end