Support resetting of Prometheus metrics between test runs

Adding the :prometheus tag to an rspec test will clear out
memory-mapped files and reset the registry.

Closes https://gitlab.com/gitlab-org/gitlab-ce/issues/39968

Gemfile

@@ -309,7 +309,7 @@ group :metrics do
   gem 'influxdb', '~> 0.2', require: false
 
   # Prometheus
-  gem 'prometheus-client-mmap', '~> 0.9.1'
+  gem 'prometheus-client-mmap', '~> 0.9.2'
   gem 'raindrops', '~> 0.18'
 end
 

Gemfile.lock

@@ -663,7 +663,7 @@ GEM
       parser
       unparser
     procto (0.0.3)
-    prometheus-client-mmap (0.9.1)
+    prometheus-client-mmap (0.9.2)
     pry (0.10.4)
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
@@ -1170,7 +1170,7 @@ DEPENDENCIES
   peek-sidekiq (~> 1.0.3)
   pg (~> 0.18.2)
   premailer-rails (~> 1.9.7)
-  prometheus-client-mmap (~> 0.9.1)
+  prometheus-client-mmap (~> 0.9.2)
   pry-byebug (~> 3.4.1)
   pry-rails (~> 0.3.4)
   rack-attack (~> 4.4.1)

doc/development/testing_guide/best_practices.md

@@ -230,6 +230,11 @@ describe "#==" do
 end
 ```
 
+### Prometheus tests
+
+Prometheus metrics may be preserved from one test run to another. To ensure that metrics are
+reset before each example, add the `:prometheus` tag to the Rspec test.
+
 ### Matchers
 
 Custom matchers should be created to clarify the intent and/or hide the

ee/spec/requests/api/geo_nodes_spec.rb

 require 'spec_helper'
 
-describe API::GeoNodes, :geo, api: true do
+describe API::GeoNodes, :geo, :prometheus, api: true do
   include ApiHelpers
   include ::EE::GeoHelpers
 
@@ -13,15 +13,6 @@ describe API::GeoNodes, :geo, api: true do
   set(:admin) { create(:admin) }
   set(:user) { create(:user) }
 
-  before do
-    # FIXME: Skip creating prometheus metrics in these specs as it interacts
-    # badly with specs in ee/spec/services/geo/metrics_update_spec.rb - we need
-    # to learn how to reliably clear prometheus state between specs.
-    #
-    # https://gitlab.com/gitlab-org/gitlab-ce/issues/39968
-    allow(Gitlab::Metrics).to receive(:prometheus_metrics_enabled?) { false }
-  end
-
   describe 'GET /geo_nodes' do
     it 'retrieves the Geo nodes if admin is logged in' do
       get api("/geo_nodes", admin)

ee/spec/services/geo/metrics_update_service_spec.rb

 require 'spec_helper'
 
-describe Geo::MetricsUpdateService, :geo do
+describe Geo::MetricsUpdateService, :geo, :prometheus do
   include ::EE::GeoHelpers
 
   set(:primary) { create(:geo_node, :primary) }

lib/gitlab/metrics/prometheus.rb

@@ -25,6 +25,14 @@ module Gitlab
           end
         end
 
+        def reset_registry!
+          clear_memoization(:registry)
+
+          REGISTRY_MUTEX.synchronize do
+            ::Prometheus::Client.reset!
+          end
+        end
+
         def registry
           strong_memoize(:registry) do
             REGISTRY_MUTEX.synchronize do

spec/lib/gitlab/metrics/prometheus_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Metrics::Prometheus, :prometheus do
+  let(:all_metrics) { Gitlab::Metrics }
+  let(:registry) { all_metrics.registry }
+
+  describe '#reset_registry!' do
+    it 'clears existing metrics' do
+      registry.counter(:test, 'test metric')
+
+      expect(registry.metrics.count).to eq(1)
+
+      all_metrics.reset_registry!
+
+      expect(all_metrics.registry.metrics.count).to eq(0)
+    end
+  end
+end

spec/spec_helper.rb

@@ -156,6 +156,13 @@ RSpec.configure do |config|
     reset_delivered_emails!
   end
 
+  config.before(:example, :prometheus) do
+    matching_files = File.join(::Prometheus::Client.configuration.multiprocess_files_dir, "*.db")
+    Dir[matching_files].map { |filename| File.delete(filename) if File.file?(filename) }
+
+    Gitlab::Metrics.reset_registry!
+  end
+
   config.around(:each, :use_clean_rails_memory_store_caching) do |example|
     caching_store = Rails.cache
     Rails.cache = ActiveSupport::Cache::MemoryStore.new