Merge branch 'jejacks0n/add-gitlab-experiment-to-robots-txt' into 'master'

Add experiment routes to robots.txt

See merge request gitlab-org/gitlab!67535

public/robots.txt

@@ -24,6 +24,7 @@ Disallow: /help
 Disallow: /s/
 Disallow: /-/profile
 Disallow: /-/ide/
+Disallow: /-/experiment
 # Restrict allowed routes to avoid very ugly search results
 Allow: /users/sign_in
 Allow: /users/*/snippets

spec/requests/robots_txt_spec.rb

@@ -28,6 +28,7 @@ RSpec.describe 'Robots.txt Requests', :aggregate_failures do
 
   it 'blocks the requests' do
     requests = [
+      Gitlab::Experiment::Configuration.mount_at,
       '/autocomplete/users',
       '/autocomplete/projects',
       '/search',