blob/auth: Teach it to handle HTTP Basic Auth too

There are cases when using user:password for /raw/... access is handy:

- when using query for auth (private_token) is not convenient for some
  reason (e.g. client processing software does not handle queries well
  when generating URLs)

- when we do not want to organize many artificial users and use their
  tokens, but instead just use per-project automatically setup

    gitlab-ci-token : <ci-token>

  artificial user & "password" which are already handled by auth backend
  for `git fetch` requests.

Handling is easy: if main auth backend rejects access, and there is
user:password in original request, we retry asking auth backend the way
as `git fetch` would do.

Access is granted if any of two ways to ask auth backend succeeds. This
way both private tokens / cookies and HTTP auth are supported.

internal/api/auth.go

@@ -43,6 +43,7 @@ type AuthCacheEntry struct {
 // Entries are keyed by project + credentials
 type AuthCacheKey struct {
 	project  string
+	userinfo string // user[:password] or ""
 	query    string // e.g. with passing in private_token=...
 	header   string // request header url-encoded, e.g. PRIVATE-TOKEN=...
 }
@@ -63,7 +64,13 @@ func NewAuthCache(a *API) *AuthCache {
 // Verify that download access is ok or not.
 // first we try to use the cache; if information is not there -> ask auth backend
 // download is ok if AuthReply.RepoPath != ""
-func (c *AuthCache) VerifyDownloadAccess(project string, query string, header http.Header) AuthReply {
+func (c *AuthCache) VerifyDownloadAccess(project string, userinfo *url.Userinfo, query string, header http.Header) AuthReply {
+	// In addition to userinfo:
+	u := ""
+	if userinfo != nil {
+		u = userinfo.String()
+	}
+
 	// Use only tokens from query/header and selected cookies to minimize cache and avoid
 	// creating redundant cache entries because of e.g. unrelated headers.
 	queryValues, _ := url.ParseQuery(query) // this is what URL.Query() does
@@ -95,7 +102,7 @@ func (c *AuthCache) VerifyDownloadAccess(project string, query string, header ht
 		h["Cookie"] = []string{hc}
 	}
 
-	key := AuthCacheKey{project, q.Encode(), h.Encode()}
+	key := AuthCacheKey{project, u, q.Encode(), h.Encode()}
 	return c.verifyDownloadAccess(key)
 }
 
@@ -194,6 +201,20 @@ func (c *AuthCache) refreshEntry(auth *AuthCacheEntry, key AuthCacheKey) {
 
 // Ask auth backend about cache key
 func (c *AuthCache) askAuthBackend(key AuthCacheKey) AuthReply {
+	// key.userinfo -> url.Userinfo
+	var user *url.Userinfo
+	if key.userinfo != "" {
+		u, err := url.Parse("http://" + key.userinfo + "@/")
+		// url prepared-to-parse userinfo must be valid
+		if err != nil {
+			panic(err)
+		}
+		if u.User == nil {
+			panic(fmt.Errorf("userinfo parse: `%s` -> empty", key.userinfo))
+		}
+		user = u.User
+	}
+
 	// key.header -> url.Values -> http.Header
 	hv, err := url.ParseQuery(key.header)
 	if err != nil {
@@ -206,7 +227,7 @@ func (c *AuthCache) askAuthBackend(key AuthCacheKey) AuthReply {
 		header[k] = v
 	}
 
-	return c.a.verifyDownloadAccess(key.project, key.query, header)
+	return c.a.verifyDownloadAccess(key.project, user, key.query, header)
 }
 
 // Ask auth backend about whether download is ok for a project.
@@ -215,11 +236,17 @@ func (c *AuthCache) askAuthBackend(key AuthCacheKey) AuthReply {
 //
 // Replies from authentication backend are cached for 30 seconds as each
 // request to Rails code is heavy and slow.
-func (a *API) VerifyDownloadAccess(project, query string, header http.Header) AuthReply {
-	return a.authCache.VerifyDownloadAccess(project, query, header)
+func (a *API) VerifyDownloadAccess(project string, user *url.Userinfo, query string, header http.Header) AuthReply {
+	return a.authCache.VerifyDownloadAccess(project, user, query, header)
+}
+
+// like Userinfo.Password(), "" if unset
+func xpassword(user *url.Userinfo) string {
+	password, _ := user.Password()
+	return password
 }
 
-func (a *API) verifyDownloadAccess(project, query string, header http.Header) AuthReply {
+func (a *API) verifyDownloadAccess(project string, user *url.Userinfo, query string, header http.Header) AuthReply {
 	authReply := AuthReply{
 		RawReply: httptest.NewRecorder(),
 	}
@@ -229,7 +256,6 @@ func (a *API) verifyDownloadAccess(project, query string, header http.Header) Au
 	//   side this supports only basic auth, not private token.
 	// - that's why we auth backend to authenticate as if it was request to
 	//   get repo archive and propagate request query and header.
-	// url := project + ".git/info/refs?service=git-upload-pack"
 	url := project + "/repository/archive.zip"
 	if query != "" {
 		url += "?" + query
@@ -239,6 +265,10 @@ func (a *API) verifyDownloadAccess(project, query string, header http.Header) Au
 		helper.Fail500(authReply.RawReply, fmt.Errorf("GET git-upload-pack: %v", err))
 		return authReply
 	}
+	if user != nil {
+		// just in case - Rails does not support HTTP Basic Auth for usual requests
+		reqDownloadAccess.SetBasicAuth(user.Username(), xpassword(user))
+	}
 	for k, v := range header {
 		reqDownloadAccess.Header[k] = v
 	}
@@ -250,5 +280,36 @@ func (a *API) verifyDownloadAccess(project, query string, header http.Header) Au
 			authReply.Response = *resp
 		}, "").ServeHTTP(authReply.RawReply, reqDownloadAccess)
 
+	// If not successful and userinfo is provided without query, retry
+	// authenticating as `git fetch` would do.
+	//
+	// The reason we want to do this second try is that HTTP auth is
+	// handled by upstream auth backend for git requests only, and we might
+	// want to use e.g. https://gitlab-ci-token:token@/.../raw/...
+	if authReply.RepoPath != "" || user == nil || query != "" {
+		return authReply
+	}
+
+	url = project + ".git/info/refs?service=git-upload-pack"
+	reqFetchAccess, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		helper.Fail500(authReply.RawReply, fmt.Errorf("GET git-upload-pack: %v", err))
+		return authReply
+	}
+	reqFetchAccess.SetBasicAuth(user.Username(), xpassword(user))
+	for k, v := range header {
+		reqFetchAccess.Header[k] = v
+	}
+
+	// reset RawReply - if failed we will return to client what this latter -
+	// - request to auth backend as `git fetch` - replies.
+	authReply.RawReply = httptest.NewRecorder()
+	a.PreAuthorizeHandler(
+		func(w http.ResponseWriter, req *http.Request, resp *Response) {
+			// if we ever get to this point - auth handler approved
+			// access and thus it is ok to download
+			authReply.Response = *resp
+		}, "").ServeHTTP(authReply.RawReply, reqFetchAccess)
+
 	return authReply
 }

internal/git/xblob.go

@@ -14,6 +14,7 @@ import (
 	"io"
 	"log"
 	"net/http"
+	"net/url"
 	"regexp"
 	"strings"
 )
@@ -39,8 +40,15 @@ func handleGetBlobRaw(a *api.API, w http.ResponseWriter, r *http.Request) {
 	project := u.Path[:rawLoc[0]]
 	refpath := u.Path[rawLoc[1]:]
 
+	// Prepare userinfo
+	var user *url.Userinfo
+	username, password, ok := r.BasicAuth()
+	if ok {
+		user = url.UserPassword(username, password)
+	}
+
 	// Query download access auth for this project
-	authReply := a.VerifyDownloadAccess(project, u.RawQuery, r.Header)
+	authReply := a.VerifyDownloadAccess(project, user, u.RawQuery, r.Header)
 	if authReply.RepoPath == "" {
 		// access denied - copy auth reply to client in full -
 		// there are HTTP code and other headers / body relevant for

main_test.go

@@ -714,11 +714,14 @@ func sha1s(data []byte) string {
 }
 
 // download an URL
-func download(t *testing.T, url string, h http.Header) (*http.Response, []byte) {
+func download(t *testing.T, url, username, password string, h http.Header) (*http.Response, []byte) {
 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
+	if !(username == "" && password == "") {
+		req.SetBasicAuth(username, password)
+	}
 	// copy header to request
 	for k, v := range h {
 		req.Header[k] = v
@@ -741,15 +744,17 @@ type DownloadContext struct {
 	t         *testing.T
 	urlPrefix string
 	Header    http.Header
+	username  string
+	password  string
 }
 
 func NewDownloadContext(t *testing.T, urlPrefix string) *DownloadContext {
 	h := make(http.Header)
-	return &DownloadContext{t, urlPrefix, h}
+	return &DownloadContext{t, urlPrefix, h, "", ""}
 }
 
 func (dl DownloadContext) download(path string) (*http.Response, []byte) {
-	return download(dl.t, dl.urlPrefix+path, dl.Header)
+	return download(dl.t, dl.urlPrefix+path, dl.username, dl.password, dl.Header)
 }
 
 // download `path` and expect content sha1 to be `expectSha1`
@@ -815,7 +820,14 @@ func TestPrivateBlobDownload(t *testing.T) {
 		token_ok2 := r.Header.Get("BBB-TOKEN") == "TOKEN-4BBB"
 		cookie, _ := r.Cookie("_gitlab_session")
 		cookie_ok3 := (cookie != nil && cookie.Value == "COOKIE-CCC")
-		if !(token_ok1 || token_ok2 || cookie_ok3) {
+		username, password, user_ok4 := r.BasicAuth()
+		if user_ok4 {
+			// user:password only accepted for `git fetch` requests
+			user_ok4 = (strings.HasSuffix(r.URL.Path, "/info/refs") &&
+				r.URL.RawQuery == "service=git-upload-pack" &&
+				username == "user-ddd" && password == "password-eee")
+		}
+		if !(token_ok1 || token_ok2 || cookie_ok3 || user_ok4) {
 			w.WriteHeader(403)
 			fmt.Fprintf(w, "Access denied")
 			return
@@ -854,4 +866,15 @@ func TestPrivateBlobDownload(t *testing.T) {
 	dl.Header.Set("Cookie", "alpha=1; _gitlab_session=COOKIE-CCC; beta=2")
 	dl.ExpectCode("/5f923865/README.md", 200)
 	dl.ExpectSha1("/5f923865/README.md", "5f7af35c185a9e5face2f4afb6d7c4f00328d04c")
+
+	dl.Header = make(http.Header) // clear
+	dl.ExpectCode("/5f923865/README.md", 403)
+	dl.username = "user-aaa"
+	dl.password = "password-bbb"
+	dl.ExpectCode("/5f923865/README.md", 403)
+	dl.username = "user-ddd"
+	dl.password = "password-eee"
+	dl.ExpectCode("/5f923865/README.md?qqq_token=1", 403)
+	dl.ExpectCode("/5f923865/README.md", 200)
+	dl.ExpectSha1("/5f923865/README.md", "5f7af35c185a9e5face2f4afb6d7c4f00328d04c")
 }