Merge branch '36836-fj-allow-indexing-snippets-in-robots' into 'master'

Resolve "Allow user snippets to be indexed"

See merge request gitlab-org/gitlab!45793

changelogs/unreleased/36836-fj-allow-indexing-snippets-in-robots.yml

+---
+title: Allow user snippets to be indexed by search crawlers
+merge_request: 45793
+author:
+type: changed

public/robots.txt

@@ -24,8 +24,9 @@ Disallow: /help
 Disallow: /s/
 Disallow: /-/profile
 Disallow: /-/ide/
-# Only specifically allow the Sign In page to avoid very ugly search results
+# Restrict allowed routes to avoid very ugly search results
 Allow: /users/sign_in
+Allow: /users/*/snippets
 
 # Generic resource routes like new, edit, raw
 # This will block routes like:

spec/requests/robots_txt_spec.rb

@@ -14,7 +14,9 @@ RSpec.describe 'Robots.txt Requests', :aggregate_failures do
   it 'allows the requests' do
     requests = [
       '/users/sign_in',
-      '/namespace/subnamespace/design.gitlab.com'
+      '/namespace/subnamespace/design.gitlab.com',
+      '/users/foo/snippets',
+      '/users/foo/snippets/1'
     ]
 
     requests.each do |request|