Merge branch 'feature/add-ci-related-metrics' into 'master'

Add CI related requests metrics

This MR adds some metrics for CI related requests.

Related to gitlab-org/gitlab-ce/#23366, gitlab-com/infrastructure#583

See merge request !97

internal/artifacts/artifacts_upload.go

@@ -9,12 +9,35 @@ import (
 	"os/exec"
 	"syscall"
 
+	"github.com/prometheus/client_golang/prometheus"
+
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/upload"
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/zipartifacts"
 )
 
+var (
+	artifactsUploadRequests = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name: "gitlab_workhorse_artifacts_upload_requests",
+			Help: "How many artifacts upload requests have been processed by gitlab-workhorse.",
+		},
+	)
+
+	artifactsUploadBytes = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name: "gitlab_workhorse_artifacts_upload_bytes",
+			Help: "How many artifacts upload bytes have been sent by gitlab-workhorse.",
+		},
+	)
+)
+
+func init() {
+	prometheus.MustRegister(artifactsUploadRequests)
+	prometheus.MustRegister(artifactsUploadBytes)
+}
+
 type artifactsUploadProcessor struct {
 	TempPath     string
 	metadataFile string
@@ -77,6 +100,11 @@ func (a *artifactsUploadProcessor) Cleanup() {
 
 func UploadArtifacts(myAPI *api.API, h http.Handler) http.Handler {
 	return myAPI.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) {
+		artifactsUploadRequests.Inc()
+		defer func() {
+			artifactsUploadBytes.Add(float64(r.ContentLength))
+		}()
+
 		if a.TempPath == "" {
 			helper.Fail500(w, r, fmt.Errorf("UploadArtifacts: TempPath is empty"))
 			return

internal/git/git-http.go

@@ -54,8 +54,12 @@ func repoPreAuthorizeHandler(myAPI *api.API, handleFunc api.HandleFunc) http.Han
 	}, "")
 }
 
-func handleGetInfoRefs(w http.ResponseWriter, r *http.Request, a *api.Response) {
-	rpc := r.URL.Query().Get("service")
+func handleGetInfoRefs(rw http.ResponseWriter, r *http.Request, a *api.Response) {
+	w := NewGitHttpResponseWriter(rw)
+	// Log 0 bytes in because we ignore the request body (and there usually is none anyway).
+	defer w.Log(r, 0)
+
+	rpc := getService(r)
 	if !(rpc == "git-upload-pack" || rpc == "git-receive-pack") {
 		// The 'dumb' Git HTTP protocol is not supported
 		http.Error(w, "Not Found", 404)
@@ -101,13 +105,18 @@ func handleGetInfoRefs(w http.ResponseWriter, r *http.Request, a *api.Response)
 	}
 }
 
-func handlePostRPC(w http.ResponseWriter, r *http.Request, a *api.Response) {
+func handlePostRPC(rw http.ResponseWriter, r *http.Request, a *api.Response) {
 	var err error
 	var body io.Reader
 	var isShallowClone bool
+	var writtenIn int64
+
+	w := NewGitHttpResponseWriter(rw)
+	defer func() {
+		w.Log(r, writtenIn)
+	}()
 
-	// Get Git action from URL
-	action := filepath.Base(r.URL.Path)
+	action := getService(r)
 	if !(action == "git-upload-pack" || action == "git-receive-pack") {
 		// The 'dumb' Git HTTP protocol is not supported
 		helper.Fail500(w, r, fmt.Errorf("handlePostRPC: unsupported action: %s", r.URL.Path))
@@ -152,7 +161,7 @@ func handlePostRPC(w http.ResponseWriter, r *http.Request, a *api.Response) {
 	defer helper.CleanUpProcessGroup(cmd) // Ensure brute force subprocess clean-up
 
 	// Write the client request body to Git's standard input
-	if _, err := io.Copy(stdin, body); err != nil {
+	if writtenIn, err = io.Copy(stdin, body); err != nil {
 		helper.Fail500(w, r, fmt.Errorf("handlePostRPC: write to %v: %v", cmd.Args, err))
 		return
 	}
@@ -182,6 +191,13 @@ func handlePostRPC(w http.ResponseWriter, r *http.Request, a *api.Response) {
 	}
 }
 
+func getService(r *http.Request) string {
+	if r.Method == "GET" {
+		return r.URL.Query().Get("service")
+	}
+	return filepath.Base(r.URL.Path)
+}
+
 func isExitError(err error) bool {
 	_, ok := err.(*exec.ExitError)
 	return ok

internal/git/response-writter.go

+package git
+
+import (
+	"net/http"
+	"strconv"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	directionIn  = "in"
+	directionOut = "out"
+)
+
+var (
+	gitHTTPSessionsActive = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "gitlab_workhorse_git_http_sessions_active",
+		Help: "Number of Git HTTP request-response cycles currently being handled by gitlab-workhorse.",
+	})
+
+	gitHTTPRequests = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "gitlab_workhorse_git_http_requests",
+			Help: "How many Git HTTP requests have been processed by gitlab-workhorse, partitioned by request type and agent.",
+		},
+		[]string{"method", "code", "service", "agent"},
+	)
+
+	gitHTTPBytes = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "gitlab_workhorse_git_http_bytes",
+			Help: "How many Git HTTP bytes have been sent by gitlab-workhorse, partitioned by request type, agent and direction.",
+		},
+		[]string{"method", "code", "service", "agent", "direction"},
+	)
+)
+
+func init() {
+	prometheus.MustRegister(gitHTTPSessionsActive)
+	prometheus.MustRegister(gitHTTPRequests)
+	prometheus.MustRegister(gitHTTPBytes)
+}
+
+type GitHttpResponseWriter struct {
+	rw      http.ResponseWriter
+	status  int
+	written int64
+}
+
+func NewGitHttpResponseWriter(rw http.ResponseWriter) *GitHttpResponseWriter {
+	gitHTTPSessionsActive.Inc()
+	return &GitHttpResponseWriter{
+		rw: rw,
+	}
+}
+
+func (w *GitHttpResponseWriter) Header() http.Header {
+	return w.rw.Header()
+}
+
+func (w *GitHttpResponseWriter) Write(data []byte) (n int, err error) {
+	if w.status == 0 {
+		w.WriteHeader(http.StatusOK)
+	}
+
+	n, err = w.rw.Write(data)
+	w.written += int64(n)
+	return n, err
+}
+
+func (w *GitHttpResponseWriter) WriteHeader(status int) {
+	if w.status != 0 {
+		return
+	}
+
+	w.status = status
+	w.rw.WriteHeader(status)
+}
+
+func (w *GitHttpResponseWriter) Log(r *http.Request, writtenIn int64) {
+	service := getService(r)
+	agent := getRequestAgent(r)
+
+	gitHTTPSessionsActive.Dec()
+	gitHTTPRequests.WithLabelValues(r.Method, strconv.Itoa(w.status), service, agent).Inc()
+	gitHTTPBytes.WithLabelValues(r.Method, strconv.Itoa(w.status), service, agent, directionIn).
+		Add(float64(writtenIn))
+	gitHTTPBytes.WithLabelValues(r.Method, strconv.Itoa(w.status), service, agent, directionOut).
+		Add(float64(w.written))
+}
+
+func getRequestAgent(r *http.Request) string {
+	u, _, ok := r.BasicAuth()
+	if !ok {
+		return "anonymous"
+	}
+
+	if u == "gitlab-ci-token" {
+		return "gitlab-ci"
+	}
+
+	return "logged"
+}

internal/helper/logging.go

@@ -93,7 +93,7 @@ func (l *loggingResponseWriter) Write(data []byte) (n int, err error) {
 	}
 	n, err = l.rw.Write(data)
 	l.written += int64(n)
-	return
+	return n, err
 }
 
 func (l *loggingResponseWriter) WriteHeader(status int) {

internal/sendfile/sendfile.go

@@ -9,12 +9,35 @@ package sendfile
 import (
 	"log"
 	"net/http"
+	"regexp"
+
+	"github.com/prometheus/client_golang/prometheus"
 
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
 )
 
 const sendFileResponseHeader = "X-Sendfile"
 
+var (
+	sendFileRequests = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "gitlab_workhorse_sendfile_requests",
+			Help: "How many X-Sendfile requests have been processed by gitlab-workhorse, partitioned by sendfile type.",
+		},
+		[]string{"type"},
+	)
+
+	sendFileBytes = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "gitlab_workhorse_sendfile_bytes",
+			Help: "How many X-Sendfile bytes have been sent by gitlab-workhorse, partitioned by sendfile type.",
+		},
+		[]string{"type"},
+	)
+
+	artifactsSendFile = regexp.MustCompile("builds/[0-9]+/artifacts")
+)
+
 type sendFileResponseWriter struct {
 	rw       http.ResponseWriter
 	status   int
@@ -22,6 +45,11 @@ type sendFileResponseWriter struct {
 	req      *http.Request
 }
 
+func init() {
+	prometheus.MustRegister(sendFileRequests)
+	prometheus.MustRegister(sendFileBytes)
+}
+
 func SendFile(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
 		s := &sendFileResponseWriter{
@@ -84,9 +112,24 @@ func sendFileFromDisk(w http.ResponseWriter, r *http.Request, file string) {
 	}
 	defer content.Close()
 
+	countSendFileMetrics(fi.Size(), r)
+
 	http.ServeContent(w, r, "", fi.ModTime(), content)
 }
 
+func countSendFileMetrics(size int64, r *http.Request) {
+	var requestType string
+	switch {
+	case artifactsSendFile.MatchString(r.RequestURI):
+		requestType = "artifacts"
+	default:
+		requestType = "other"
+	}
+
+	sendFileRequests.WithLabelValues(requestType).Inc()
+	sendFileBytes.WithLabelValues(requestType).Add(float64(size))
+}
+
 func (s *sendFileResponseWriter) Flush() {
 	s.WriteHeader(http.StatusOK)
 }