Wrap the experiment api in an experiment

- This is intended to capture if the experiment system is broken or not
working as intended — in a way that will hopefully surface to the
experiment team in an obvious way.
- This also is much more explicit in how we test the gitlab_experiment
iglu/snowplow context.

.gitlab/CODEOWNERS

@@ -277,6 +277,7 @@ Dangerfile @gl-quality/eng-prod
 /lib/gitlab/experimentation/ @gitlab-org/growth/experiment-devs
 /lib/gitlab/experimentation.rb @gitlab-org/growth/experiment-devs
 /lib/gitlab/experimentation_logger.rb @gitlab-org/growth/experiment-devs
+/ee/spec/requests/api/experiments_spec.rb @gitlab-org/growth/experiment-devs
 
 [Legal]
 /config/dependency_decisions.yml @gitlab-org/legal-reviewers

ee/lib/api/experiments.rb

@@ -11,13 +11,20 @@ module API
         success EE::API::Entities::Experiment
       end
       get do
-        experiments = Feature::Definition.definitions.values.select { |d| d.attributes[:type] == 'experiment' }
+        experiments = []
+
+        experiment(:null_hypothesis, canary: true, user: current_user) do |e|
+          e.use { bad_request! 'experimentation may not be working right now' }
+          e.try { experiments = Feature::Definition.definitions.values.select { |d| d.attributes[:type] == 'experiment' } }
+        end
 
         present experiments, with: EE::API::Entities::Experiment, current_user: current_user
       end
     end
 
     helpers do
+      include Gitlab::Experiment::Dsl
+
       def authorize_read_experiments!
         authenticate!
 

ee/spec/requests/api/experiments_spec.rb

@@ -12,12 +12,13 @@ RSpec.describe API::Experiments do
     context 'when on .com' do
       before do
         allow(Gitlab).to receive(:com?).and_return(true)
+        stub_experiments(null_hypothesis: :candidate)
 
         definition = YAML.load_file(definition_yaml).deep_symbolize_keys!
         allow(Feature::Definition.definitions).to receive(:values).and_return([
           Feature::Definition.new(definition_yaml.to_s, definition),
           Feature::Definition.new(
-            "foo/non_experiment.yml",
+            'foo/non_experiment.yml',
             definition.merge(type: 'development', name: 'non_experiment')
           )
         ])
@@ -54,15 +55,15 @@ RSpec.describe API::Experiments do
 
           expect(response).to have_gitlab_http_status(:ok)
           expect(json_response).to include({
-            key: "null_hypothesis",
+            key: 'null_hypothesis',
             state: :off,
             enabled: false,
-            name: "null_hypothesis",
-            introduced_by_url: "https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45840",
+            name: 'null_hypothesis',
+            introduced_by_url: 'https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45840',
             rollout_issue_url: nil,
-            milestone: "13.7",
-            type: "experiment",
-            group: "group::adoption",
+            milestone: '13.7',
+            type: 'experiment',
+            group: 'group::adoption',
             default_enabled: false
           }.as_json)
         end
@@ -76,9 +77,83 @@ RSpec.describe API::Experiments do
           expect(json_response).to include(hash_including({
             state: :conditional,
             enabled: true,
-            name: "null_hypothesis"
+            name: 'null_hypothesis'
           }.as_json))
         end
+
+        describe 'the null_hypothesis as a canary' do
+          # This group of test ensures that we will continue to have a functional
+          # backend experiment. It's part of the suite of tooling that's in place to
+          # ensure that we don't change notable aspects of experimentation, like
+          # the position of the contexts etc.
+          #
+          # We wrap our experiments endpoint in a canary like experiment that if
+          # broken will render this endpoint visibly broken.
+          #
+          # This is something of an integration level and shouldn't be adjusted
+          # without proper consultation with the relevant Growth teams.
+
+          it 'runs and tracks the expected events' do
+            contexts = []
+
+            # Yes, we really do want to test this and the only way to get here
+            # is by calling a private method.
+            expect(Gitlab::Tracking.send(:snowplow)).to receive(:event).with(
+              'null_hypothesis',
+              'assignment',
+              label: nil,
+              property: nil,
+              value: nil,
+              context: [
+                instance_of(SnowplowTracker::SelfDescribingJson),
+                instance_of(SnowplowTracker::SelfDescribingJson)
+              ]
+            ) { |_, _, **options| contexts = options[:context] }
+
+            get api('/experiments', user)
+
+            # Ensure the order of the contexts stays correct for now.
+            #
+            # If you change this, you need to talk with the growth team,
+            # because some reporting is done (incorrectly) based on the index
+            # of this context.
+            expect(contexts[1].to_json).to include({
+              schema: 'iglu:com.gitlab/gitlab_experiment/jsonschema/1-0-0',
+              data: {
+                experiment: 'null_hypothesis',
+                key: anything,
+                variant: 'candidate'
+              }
+            })
+          end
+
+          it 'returns a 400 if experimentation seems broken' do
+            # we assume that rendering control would only be done in error.
+            stub_experiments(null_hypothesis: :control)
+
+            get api('/experiments', user)
+
+            expect(response).to have_gitlab_http_status(:bad_request)
+            expect(json_response).to eq({
+              message: '400 Bad request - experimentation may not be working right now'
+            }.as_json)
+          end
+
+          it 'publishes into a collection of experiments that have been run in the request' do
+            pending 'requires gitlab-experiment >= 0.5.4'
+
+            get api('/experiments', user)
+
+            expect(ApplicationExperiment.published_experiments).to eq(
+              'null_hypothesis' => {
+                excluded: false,
+                experiment: 'null_hypothesis',
+                key: 'abc123',
+                variant: 'candidate'
+              }
+            )
+          end
+        end
       end
     end
 

spec/features/admin/admin_abuse_reports_spec.rb

@@ -39,18 +39,18 @@ RSpec.describe "Admin::AbuseReports", :js do
           expect(page).to have_content('Abuse Reports')
 
           published_experiments = page.evaluate_script('window.gon.experiment')
-          expect(published_experiments).to eq(
-            null_hypothesis: {
-              experiment: 'null_hypothesis',
-              key: 'c934a2885104c464e13bfae61b7d9dc2',
-              variant: 'candidate'
-            }.as_json
-          )
+          expect(published_experiments).to include({
+            'null_hypothesis' => {
+              'experiment' => 'null_hypothesis',
+              'key' => anything,
+              'variant' => 'candidate'
+            }
+          })
         end
       end
 
       describe 'in the abuse report view' do
-        it 'presents information about abuse report', :experiment do
+        it 'presents information about abuse report' do
           visit admin_abuse_reports_path
 
           expect(page).to have_content('Abuse Reports')
@@ -90,7 +90,7 @@ RSpec.describe "Admin::AbuseReports", :js do
 
     describe 'filtering by user' do
       let!(:user2) { create(:user) }
-      let!(:abuse_report)   { create(:abuse_report, user: user) }
+      let!(:abuse_report) { create(:abuse_report, user: user) }
       let!(:abuse_report_2) { create(:abuse_report, user: user2) }
 
       it 'shows only single user report' do