Merge branch 'raw-blob' into 'master'

Blobs via workhorse

Works together with https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/2451

See merge request !30

internal/api/api.go

@@ -3,7 +3,6 @@ package api
 import (
 	"../badgateway"
 	"../helper"
-	"../proxy"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -97,7 +96,7 @@ func (api *API) newRequest(r *http.Request, body io.Reader, suffix string) (*htt
 	authReq := &http.Request{
 		Method: r.Method,
 		URL:    rebaseUrl(r.URL, api.URL, suffix),
-		Header: proxy.HeaderClone(r.Header),
+		Header: helper.HeaderClone(r.Header),
 	}
 	if body != nil {
 		authReq.Body = ioutil.NopCloser(body)

internal/git/blob.go

+package git
+
+import (
+	"../helper"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"strings"
+)
+
+type blobParams struct {
+	RepoPath string
+	BlobId   string
+}
+
+const SendBlobPrefix = "git-blob:"
+
+func SendBlob(w http.ResponseWriter, r *http.Request, sendData string) {
+	params, err := unpackSendData(sendData)
+	if err != nil {
+		helper.Fail500(w, fmt.Errorf("SendBlob: unpack sendData: %v", err))
+		return
+	}
+	log.Printf("SendBlob: sending %q for %q", params.BlobId, r.URL.Path)
+
+	gitShowCmd := gitCommand("", "git", "--git-dir="+params.RepoPath, "cat-file", "blob", params.BlobId)
+	stdout, err := gitShowCmd.StdoutPipe()
+	if err != nil {
+		helper.Fail500(w, fmt.Errorf("SendBlob: git  stdout: %v", err))
+		return
+	}
+	if err := gitShowCmd.Start(); err != nil {
+		helper.Fail500(w, fmt.Errorf("SendBlob: start %v: %v", gitShowCmd, err))
+		return
+	}
+	defer helper.CleanUpProcessGroup(gitShowCmd)
+
+	if _, err := io.Copy(w, stdout); err != nil {
+		helper.LogError(fmt.Errorf("SendBlob: copy git cat-file stdout: %v", err))
+		return
+	}
+	if err := gitShowCmd.Wait(); err != nil {
+		helper.LogError(fmt.Errorf("SendBlob: wait for git cat-file: %v", err))
+		return
+	}
+}
+
+func unpackSendData(sendData string) (*blobParams, error) {
+	jsonBytes, err := base64.URLEncoding.DecodeString(strings.TrimPrefix(sendData, SendBlobPrefix))
+	if err != nil {
+		return nil, err
+	}
+	result := &blobParams{}
+	if err := json.Unmarshal([]byte(jsonBytes), result); err != nil {
+		return nil, err
+	}
+	return result, nil
+}

internal/helper/helpers.go

@@ -72,6 +72,16 @@ func HTTPError(w http.ResponseWriter, r *http.Request, error string, code int) {
 	http.Error(w, error, code)
 }
 
+func HeaderClone(h http.Header) http.Header {
+	h2 := make(http.Header, len(h))
+	for k, vv := range h {
+		vv2 := make([]string, len(vv))
+		copy(vv2, vv)
+		h2[k] = vv2
+	}
+	return h2
+}
+
 func CleanUpProcessGroup(cmd *exec.Cmd) {
 	if cmd == nil {
 		return

internal/proxy/proxy.go

@@ -2,6 +2,8 @@ package proxy
 
 import (
 	"../badgateway"
+	"../helper"
+	"../senddata"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
@@ -25,24 +27,14 @@ func NewProxy(myURL *url.URL, version string, roundTripper *badgateway.RoundTrip
 	return &p
 }
 
-func HeaderClone(h http.Header) http.Header {
-	h2 := make(http.Header, len(h))
-	for k, vv := range h {
-		vv2 := make([]string, len(vv))
-		copy(vv2, vv)
-		h2[k] = vv2
-	}
-	return h2
-}
-
 func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// Clone request
 	req := *r
-	req.Header = HeaderClone(r.Header)
+	req.Header = helper.HeaderClone(r.Header)
 
 	// Set Workhorse version
 	req.Header.Set("Gitlab-Workhorse", p.Version)
-	rw := newSendFileResponseWriter(w, &req)
+	rw := senddata.NewSendFileResponseWriter(w, &req)
 	defer rw.Flush()
 
 	p.reverseProxy.ServeHTTP(&rw, &req)

internal/proxy/sendfile.go → internal/senddata/sendfile.go

@@ -4,12 +4,19 @@ via the X-Sendfile mechanism. All that is needed in the Rails code is the
 'send_file' method.
 */
 
-package proxy
+package senddata
 
 import (
+	"../git"
 	"../helper"
 	"log"
 	"net/http"
+	"strings"
+)
+
+const (
+	sendDataResponseHeader = "Gitlab-Workhorse-Send-Data"
+	sendFileResponseHeader = "X-Sendfile"
 )
 
 type sendFileResponseWriter struct {
@@ -19,11 +26,12 @@ type sendFileResponseWriter struct {
 	req      *http.Request
 }
 
-func newSendFileResponseWriter(rw http.ResponseWriter, req *http.Request) sendFileResponseWriter {
+func NewSendFileResponseWriter(rw http.ResponseWriter, req *http.Request) sendFileResponseWriter {
 	s := sendFileResponseWriter{
 		rw:  rw,
 		req: req,
 	}
+	// Advertise to upstream (Rails) that we support X-Sendfile
 	req.Header.Set("X-Sendfile-Type", "X-Sendfile")
 	return s
 }
@@ -48,30 +56,41 @@ func (s *sendFileResponseWriter) WriteHeader(status int) {
 	}
 
 	s.status = status
+	if s.status != http.StatusOK {
+		s.rw.WriteHeader(s.status)
+		return
+	}
 
-	// Check X-Sendfile header
-	file := s.Header().Get("X-Sendfile")
-	s.Header().Del("X-Sendfile")
+	if file := s.Header().Get(sendFileResponseHeader); file != "" {
+		s.Header().Del(sendFileResponseHeader)
+		// Mark this connection as hijacked
+		s.hijacked = true
 
-	// If file is empty or status is not 200 pass through header
-	if file == "" || s.status != http.StatusOK {
-		s.rw.WriteHeader(s.status)
+		// Serve the file
+		sendFileFromDisk(s.rw, s.req, file)
+		return
+	}
+	if sendData := s.Header().Get(sendDataResponseHeader); strings.HasPrefix(sendData, git.SendBlobPrefix) {
+		s.Header().Del(sendDataResponseHeader)
+		s.hijacked = true
+		git.SendBlob(s.rw, s.req, sendData)
 		return
 	}
 
-	// Mark this connection as hijacked
-	s.hijacked = true
+	s.rw.WriteHeader(s.status)
+	return
+}
 
-	// Serve the file
-	log.Printf("Send file %q for %s %q", file, s.req.Method, s.req.RequestURI)
+func sendFileFromDisk(w http.ResponseWriter, r *http.Request, file string) {
+	log.Printf("Send file %q for %s %q", file, r.Method, r.RequestURI)
 	content, fi, err := helper.OpenFile(file)
 	if err != nil {
-		http.NotFound(s.rw, s.req)
+		http.NotFound(w, r)
 		return
 	}
 	defer content.Close()
 
-	http.ServeContent(s.rw, s.req, "", fi.ModTime(), content)
+	http.ServeContent(w, r, "", fi.ModTime(), content)
 }
 
 func (s *sendFileResponseWriter) Flush() {

main_test.go

@@ -555,6 +555,48 @@ func TestArtifactsGetSingleFile(t *testing.T) {
 	}
 }
 
+func TestGetGitBlob(t *testing.T) {
+	blobId := "50b27c6518be44c42c4d87966ae2481ce895624c" // the LICENSE file in the test repository
+	blobLength := 1075
+	headerKey := http.CanonicalHeaderKey("Gitlab-Workhorse-Send-Data")
+	ts := testhelper.TestServerWithHandler(regexp.MustCompile(`.`), func(w http.ResponseWriter, r *http.Request) {
+		responseJSON := fmt.Sprintf(`{"RepoPath":"%s","BlobId":"%s"}`, path.Join(testRepoRoot, testRepo), blobId)
+		encodedJSON := base64.StdEncoding.EncodeToString([]byte(responseJSON))
+		w.Header().Set(headerKey, "git-blob:"+encodedJSON)
+		// Prevent the Go HTTP server from setting the Content-Length to 0.
+		w.Header().Set("Transfer-Encoding", "chunked")
+		if _, err := fmt.Fprintf(w, "GNU General Public License"); err != nil {
+			t.Fatal(err)
+		}
+		return
+	})
+	defer ts.Close()
+	ws := startWorkhorseServer(ts.URL)
+	defer ws.Close()
+	resourcePath := "/something"
+	resp, err := http.Get(ws.URL + resourcePath)
+	if err != nil {
+		t.Error(err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		t.Errorf("GET %q: expected 200, got %d", resourcePath, resp.StatusCode)
+	}
+	if len(resp.Header[headerKey]) != 0 {
+		t.Fatalf("Unexpected response header: %s: %q", headerKey, resp.Header.Get(headerKey))
+	}
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(body) != blobLength {
+		t.Fatalf("Expected body of %d bytes, got %d", blobLength, len(body))
+	}
+	if !strings.HasPrefix(string(body), "The MIT License (MIT)") {
+		t.Fatalf("Expected MIT license, got %q", body)
+	}
+}
+
 func setupStaticFile(fpath, content string) error {
 	cwd, err := os.Getwd()
 	if err != nil {