Use ProxyVariableSubstitutionService in alerts

Use the ProxyVariableSubstitutionService to substitute variables in
alert queries. This puts alerts on par with metrics. It also ensures
that any change in variable substitution logic is caught by alert specs.

app/services/clusters/applications/prometheus_config_service.rb

@@ -132,19 +132,21 @@ module Clusters
       end
 
       def alerts(environment)
-        variables = Gitlab::Prometheus::QueryVariables.call(environment)
         alerts = Projects::Prometheus::AlertsFinder
           .new(environment: environment)
           .execute
 
         alerts.map do |alert|
-          substitute_query_variables(alert.to_param, variables)
+          hash = alert.to_param
+          hash['expr'] = substitute_query_variables(hash['expr'], environment)
+          hash
         end
       end
 
-      def substitute_query_variables(hash, variables)
-        hash['expr'] %= variables
-        hash
+      def substitute_query_variables(query, environment)
+        result = ::Prometheus::ProxyVariableSubstitutionService.new(environment, query: query).execute
+
+        result[:params][:query]
       end
 
       def environments

changelogs/unreleased/220144-substitute-variables-in-alerts.yml

+---
+title: Fix bug with variable substitution in alerts
+merge_request: 33772
+author:
+type: fixed

spec/services/clusters/applications/prometheus_config_service_spec.rb

@@ -90,23 +90,25 @@ describe Clusters::Applications::PrometheusConfigService do
             create(:prometheus_alert,
                    project: project,
                    environment: production,
-                   prometheus_metric: metric)
+                   prometheus_metric: metric,
+                   operator: PrometheusAlert.operators['gt'],
+                   threshold: 0)
           end
 
           let(:metric) do
             create(:prometheus_metric, query: query, project: project)
           end
 
-          let(:query) { '%{ci_environment_slug}' }
+          let(:query) { 'up{environment="{{ci_environment_slug}}"}' }
 
           it 'substitutes query variables' do
             expect(Gitlab::Prometheus::QueryVariables)
               .to receive(:call)
-              .with(production)
+              .with(production, start_time: nil, end_time: nil)
               .and_call_original
 
             expr = groups.dig(0, 'rules', 0, 'expr')
-            expect(expr).to include(production.name)
+            expect(expr).to eq("up{environment=\"#{production.slug}\"} > 0.0")
           end
         end
 
@@ -127,13 +129,15 @@ describe Clusters::Applications::PrometheusConfigService do
           end
 
           it 'substitutes query variables once per environment' do
+            allow(Gitlab::Prometheus::QueryVariables).to receive(:call).and_call_original
+
             expect(Gitlab::Prometheus::QueryVariables)
               .to receive(:call)
-              .with(production)
+              .with(production, start_time: nil, end_time: nil)
 
             expect(Gitlab::Prometheus::QueryVariables)
               .to receive(:call)
-              .with(staging)
+              .with(staging, start_time: nil, end_time: nil)
 
             subject
           end