godoc: serve robots.txt raw

R=gri, niemeyer, rsc, r2, r
CC=golang-dev
https://golang.org/cl/4188063

robots.txt

+User-agent: *
+Disallow: /

src/cmd/godoc/godoc.go

@@ -91,6 +91,7 @@ func registerPublicHandlers(mux *http.ServeMux) {
 	mux.Handle(pkgHandler.pattern, &pkgHandler)
 	mux.HandleFunc("/doc/codewalk/", codewalk)
 	mux.HandleFunc("/search", search)
+	mux.Handle("/robots.txt", fileServer)
 	mux.HandleFunc("/", serveFile)
 }