Merge branch '273148-fj-disallow-some-project-routes' into 'master'

Disallow some project routes

See merge request gitlab-org/gitlab!46218

changelogs/unreleased/273148-fj-disallow-some-project-routes.yml

+---
+title: Disallow some project routes in robots.txt
+merge_request: 46218
+author:
+type: changed

public/robots.txt

@@ -67,3 +67,16 @@ Disallow: /*/protected_branches
 Disallow: /*/uploads/
 Disallow: /*/project_members
 Disallow: /*/settings
+Disallow: /*/-/import
+Disallow: /*/-/environments
+Disallow: /*/-/jobs
+Disallow: /*/-/requirements_management
+Disallow: /*/-/pipelines
+Disallow: /*/-/pipeline_schedules
+Disallow: /*/-/dependencies
+Disallow: /*/-/licenses
+Disallow: /*/-/metrics
+Disallow: /*/-/incidents
+Disallow: /*/-/value_stream_analytics
+Disallow: /*/-/analytics
+Disallow: /*/insights

spec/requests/robots_txt_spec.rb

@@ -66,7 +66,20 @@ RSpec.describe 'Robots.txt Requests', :aggregate_failures do
       '/foo/bar/uploads/foo',
       '/foo/bar/project_members',
       '/foo/bar/settings',
-      '/namespace/subnamespace/design.gitlab.com/settings'
+      '/namespace/subnamespace/design.gitlab.com/settings',
+      '/foo/bar/-/import',
+      '/foo/bar/-/environments',
+      '/foo/bar/-/jobs',
+      '/foo/bar/-/requirements_management',
+      '/foo/bar/-/pipelines',
+      '/foo/bar/-/pipeline_schedules',
+      '/foo/bar/-/dependencies',
+      '/foo/bar/-/licenses',
+      '/foo/bar/-/metrics',
+      '/foo/bar/-/incidents',
+      '/foo/bar/-/value_stream_analytics',
+      '/foo/bar/-/analytics',
+      '/foo/bar/insights'
     ]
 
     requests.each do |request|