Merge branch 'artifacts' into 'master'

Implement multipart form rewriting to support upload offloading

1. This parses multipart form data and saves all found files as files in TempPath. TempPath is received from Rails by calling authorize request. The rewritten multipart form data contains `file.path` where the temporary file is stored, and `file.name` the original name of file as stored in Content-Disposition. The temporary file is removed afterwards, if it's not consumed by GitLab Rails. If the body is not multipart, forward request.

2. All artifacts downloads are offloaded by exposing X-Sendfile-Type extension.


See merge request !5

artifacts.go

+package main
+
+func artifactsAuthorizeHandler(handleFunc serviceHandleFunc) serviceHandleFunc {
+	return preAuthorizeHandler(handleFunc, "/authorize")
+}

authorization_test.go

+package main
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"regexp"
+	"testing"
+)
+
+func okHandler(w http.ResponseWriter, r *gitRequest) {
+	w.WriteHeader(201)
+	fmt.Fprint(w, "{\"status\":\"ok\"}")
+}
+
+func runPreAuthorizeHandler(t *testing.T, suffix string, url *regexp.Regexp, authorizationResponse interface{}, returnCode, expectedCode int) *httptest.ResponseRecorder {
+	// Prepare test server and backend
+	ts := testAuthServer(url, returnCode, authorizationResponse)
+	defer ts.Close()
+
+	// Create http request
+	httpRequest, err := http.NewRequest("GET", "/address", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	request := gitRequest{
+		Request: httpRequest,
+		u:       newUpstream(ts.URL, nil),
+	}
+
+	response := httptest.NewRecorder()
+	preAuthorizeHandler(okHandler, suffix)(response, &request)
+	assertResponseCode(t, response, expectedCode)
+	return response
+}
+
+func TestPreAuthorizeHappyPath(t *testing.T) {
+	runPreAuthorizeHandler(
+		t, "/authorize",
+		regexp.MustCompile(`/authorize\z`),
+		&authorizationResponse{},
+		200, 201)
+}
+
+func TestPreAuthorizeSuffix(t *testing.T) {
+	runPreAuthorizeHandler(
+		t, "/different-authorize",
+		regexp.MustCompile(`/authorize\z`),
+		&authorizationResponse{},
+		200, 404)
+}
+
+func TestPreAuthorizeJsonFailure(t *testing.T) {
+	runPreAuthorizeHandler(
+		t, "/authorize",
+		regexp.MustCompile(`/authorize\z`),
+		"not-json",
+		200, 500)
+}

git-http.go

@@ -5,7 +5,6 @@ In this file we handle the Git 'smart HTTP' protocol
 package main
 
 import (
-	"compress/gzip"
 	"fmt"
 	"io"
 	"net/http"
@@ -58,7 +57,6 @@ func handleGetInfoRefs(w http.ResponseWriter, r *gitRequest) {
 }
 
 func handlePostRPC(w http.ResponseWriter, r *gitRequest) {
-	var body io.ReadCloser
 	var err error
 
 	// Get Git action from URL
@@ -69,18 +67,6 @@ func handlePostRPC(w http.ResponseWriter, r *gitRequest) {
 		return
 	}
 
-	// The client request body may have been gzipped.
-	if r.Header.Get("Content-Encoding") == "gzip" {
-		body, err = gzip.NewReader(r.Body)
-		if err != nil {
-			fail500(w, "handlePostRPC", err)
-			return
-		}
-	} else {
-		body = r.Body
-	}
-	defer body.Close()
-
 	// Prepare our Git subprocess
 	cmd := gitCommand(r.GL_ID, "git", subCommand(action), "--stateless-rpc", r.RepoPath)
 	stdout, err := cmd.StdoutPipe()
@@ -102,7 +88,7 @@ func handlePostRPC(w http.ResponseWriter, r *gitRequest) {
 	defer cleanUpProcessGroup(cmd) // Ensure brute force subprocess clean-up
 
 	// Write the client request body to Git's standard input
-	if _, err := io.Copy(stdin, body); err != nil {
+	if _, err := io.Copy(stdin, r.Body); err != nil {
 		fail500(w, "handlePostRPC write to subprocess", err)
 		return
 	}
@@ -112,9 +98,6 @@ func handlePostRPC(w http.ResponseWriter, r *gitRequest) {
 	// It may take a while before we return and the deferred closes happen
 	// so let's free up some resources already.
 	r.Body.Close()
-	// If the body was compressed, body != r.Body and this frees up the
-	// gzip.Reader.
-	body.Close()
 
 	// Start writing the response
 	w.Header().Add("Content-Type", fmt.Sprintf("application/x-%s-result", action))

handlers.go

+package main
+
+import (
+	"compress/gzip"
+	"fmt"
+	"io"
+	"net/http"
+)
+
+func contentEncodingHandler(handleFunc serviceHandleFunc) serviceHandleFunc {
+	return func(w http.ResponseWriter, r *gitRequest) {
+		var body io.ReadCloser
+		var err error
+
+		// The client request body may have been gzipped.
+		contentEncoding := r.Header.Get("Content-Encoding")
+		switch contentEncoding {
+		case "":
+			body = r.Body
+		case "gzip":
+			body, err = gzip.NewReader(r.Body)
+		default:
+			err = fmt.Errorf("unsupported content encoding: %s", contentEncoding)
+		}
+
+		if err != nil {
+			fail500(w, "contentEncodingHandler", err)
+			return
+		}
+		defer body.Close()
+
+		r.Body = body
+		r.Header.Del("Content-Encoding")
+
+		handleFunc(w, r)
+	}
+}

handlers_test.go

+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"testing"
+)
+
+func TestGzipEncoding(t *testing.T) {
+	resp := httptest.NewRecorder()
+
+	var b bytes.Buffer
+	w := gzip.NewWriter(&b)
+	fmt.Fprint(w, "test")
+	w.Close()
+
+	body := ioutil.NopCloser(&b)
+
+	req, err := http.NewRequest("POST", "http://address/test", body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	req.Header.Set("Content-Encoding", "gzip")
+
+	request := gitRequest{Request: req}
+	contentEncodingHandler(func(w http.ResponseWriter, r *gitRequest) {
+		if _, ok := r.Body.(*gzip.Reader); !ok {
+			t.Fatal("Expected gzip reader for body, but it's:", reflect.TypeOf(r.Body))
+		}
+		if r.Header.Get("Content-Encoding") != "" {
+			t.Fatal("Content-Encoding should be deleted")
+		}
+	})(resp, &request)
+
+	assertResponseCode(t, resp, 200)
+}
+
+func TestNoEncoding(t *testing.T) {
+	resp := httptest.NewRecorder()
+
+	var b bytes.Buffer
+	body := ioutil.NopCloser(&b)
+
+	req, err := http.NewRequest("POST", "http://address/test", body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	req.Header.Set("Content-Encoding", "")
+
+	request := gitRequest{Request: req}
+	contentEncodingHandler(func(w http.ResponseWriter, r *gitRequest) {
+		if r.Body != body {
+			t.Fatal("Expected the same body")
+		}
+		if r.Header.Get("Content-Encoding") != "" {
+			t.Fatal("Content-Encoding should be deleted")
+		}
+	})(resp, &request)
+
+	assertResponseCode(t, resp, 200)
+}
+
+func TestInvalidEncoding(t *testing.T) {
+	resp := httptest.NewRecorder()
+
+	req, err := http.NewRequest("POST", "http://address/test", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	req.Header.Set("Content-Encoding", "application/unknown")
+
+	request := gitRequest{Request: req}
+	contentEncodingHandler(func(w http.ResponseWriter, r *gitRequest) {
+		t.Fatal("it shouldn't be executed")
+	})(resp, &request)
+
+	assertResponseCode(t, resp, 500)
+}

helpers.go

@@ -6,13 +6,22 @@ package main
 
 import (
 	"fmt"
+	"io"
+	"io/ioutil"
 	"log"
 	"net/http"
+	"net/url"
 	"os"
 	"os/exec"
+	"strings"
 	"syscall"
 )
 
+func fail400(w http.ResponseWriter, context string, err error) {
+	http.Error(w, "Bad request", 400)
+	logContext(context, err)
+}
+
 func fail500(w http.ResponseWriter, context string, err error) {
 	http.Error(w, "Internal server error", 500)
 	logContext(context, err)
@@ -52,3 +61,21 @@ func cleanUpProcessGroup(cmd *exec.Cmd) {
 	// reap our child process
 	cmd.Wait()
 }
+
+func forwardResponseToClient(w http.ResponseWriter, r *http.Response) {
+	log.Printf("PROXY:%s %q %d", r.Request.Method, r.Request.URL, r.StatusCode)
+
+	for k, v := range r.Header {
+		w.Header()[k] = v
+	}
+
+	w.WriteHeader(r.StatusCode)
+	io.Copy(w, r.Body)
+}
+
+func setHttpPostForm(r *http.Request, values url.Values) {
+	dataBuffer := strings.NewReader(values.Encode())
+	r.Body = ioutil.NopCloser(dataBuffer)
+	r.ContentLength = int64(dataBuffer.Len())
+	r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+}

helpers_test.go

+package main
+
+import (
+	"net/http/httptest"
+	"testing"
+)
+
+func assertResponseCode(t *testing.T, response *httptest.ResponseRecorder, expectedCode int) {
+	if response.Code != expectedCode {
+		t.Fatalf("for HTTP request expected to get %d, got %d instead", expectedCode, response.Code)
+	}
+}

main_test.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"log"
@@ -10,6 +11,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"regexp"
 	"testing"
 	"time"
 )
@@ -29,7 +31,7 @@ func TestAllowedClone(t *testing.T) {
 	}
 
 	// Prepare test server and backend
-	ts := testAuthServer(200, gitOkBody(t))
+	ts := testAuthServer(nil, 200, gitOkBody(t))
 	defer ts.Close()
 	ws := startWorkhorseServer(ts.URL)
 	defer ws.Close()
@@ -51,7 +53,7 @@ func TestDeniedClone(t *testing.T) {
 	}
 
 	// Prepare test server and backend
-	ts := testAuthServer(403, "Access denied")
+	ts := testAuthServer(nil, 403, "Access denied")
 	defer ts.Close()
 	ws := startWorkhorseServer(ts.URL)
 	defer ws.Close()
@@ -69,7 +71,7 @@ func TestAllowedPush(t *testing.T) {
 	preparePushRepo(t)
 
 	// Prepare the test server and backend
-	ts := testAuthServer(200, gitOkBody(t))
+	ts := testAuthServer(nil, 200, gitOkBody(t))
 	defer ts.Close()
 	ws := startWorkhorseServer(ts.URL)
 	defer ws.Close()
@@ -84,7 +86,7 @@ func TestDeniedPush(t *testing.T) {
 	preparePushRepo(t)
 
 	// Prepare the test server and backend
-	ts := testAuthServer(403, "Access denied")
+	ts := testAuthServer(nil, 403, "Access denied")
 	defer ts.Close()
 	ws := startWorkhorseServer(ts.URL)
 	defer ws.Close()
@@ -104,7 +106,7 @@ func TestAllowedDownloadZip(t *testing.T) {
 
 	// Prepare test server and backend
 	archiveName := "foobar.zip"
-	ts := testAuthServer(200, archiveOkBody(t, archiveName))
+	ts := testAuthServer(nil, 200, archiveOkBody(t, archiveName))
 	defer ts.Close()
 	ws := startWorkhorseServer(ts.URL)
 	defer ws.Close()
@@ -123,7 +125,7 @@ func TestAllowedDownloadTar(t *testing.T) {
 
 	// Prepare test server and backend
 	archiveName := "foobar.tar"
-	ts := testAuthServer(200, archiveOkBody(t, archiveName))
+	ts := testAuthServer(nil, 200, archiveOkBody(t, archiveName))
 	defer ts.Close()
 	ws := startWorkhorseServer(ts.URL)
 	defer ws.Close()
@@ -142,7 +144,7 @@ func TestAllowedDownloadTarGz(t *testing.T) {
 
 	// Prepare test server and backend
 	archiveName := "foobar.tar.gz"
-	ts := testAuthServer(200, archiveOkBody(t, archiveName))
+	ts := testAuthServer(nil, 200, archiveOkBody(t, archiveName))
 	defer ts.Close()
 	ws := startWorkhorseServer(ts.URL)
 	defer ws.Close()
@@ -161,7 +163,7 @@ func TestAllowedDownloadTarBz2(t *testing.T) {
 
 	// Prepare test server and backend
 	archiveName := "foobar.tar.bz2"
-	ts := testAuthServer(200, archiveOkBody(t, archiveName))
+	ts := testAuthServer(nil, 200, archiveOkBody(t, archiveName))
 	defer ts.Close()
 	ws := startWorkhorseServer(ts.URL)
 	defer ws.Close()
@@ -180,7 +182,7 @@ func TestAllowedApiDownloadZip(t *testing.T) {
 
 	// Prepare test server and backend
 	archiveName := "foobar.zip"
-	ts := testAuthServer(200, archiveOkBody(t, archiveName))
+	ts := testAuthServer(nil, 200, archiveOkBody(t, archiveName))
 	defer ts.Close()
 	ws := startWorkhorseServer(ts.URL)
 	defer ws.Close()
@@ -199,7 +201,7 @@ func TestDownloadCacheHit(t *testing.T) {
 
 	// Prepare test server and backend
 	archiveName := "foobar.zip"
-	ts := testAuthServer(200, archiveOkBody(t, archiveName))
+	ts := testAuthServer(nil, 200, archiveOkBody(t, archiveName))
 	defer ts.Close()
 	ws := startWorkhorseServer(ts.URL)
 	defer ws.Close()
@@ -230,7 +232,7 @@ func TestDownloadCacheCreate(t *testing.T) {
 
 	// Prepare test server and backend
 	archiveName := "foobar.zip"
-	ts := testAuthServer(200, archiveOkBody(t, archiveName))
+	ts := testAuthServer(nil, 200, archiveOkBody(t, archiveName))
 	defer ts.Close()
 	ws := startWorkhorseServer(ts.URL)
 	defer ws.Close()
@@ -281,12 +283,47 @@ func newBranch() string {
 	return fmt.Sprintf("branch-%d", time.Now().UnixNano())
 }
 
-func testAuthServer(code int, body string) *httptest.Server {
+func testServerWithHandler(url *regexp.Regexp, handler http.HandlerFunc) *httptest.Server {
 	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if url != nil && !url.MatchString(r.URL.Path) {
+			log.Println("UPSTREAM", r.Method, r.URL, "DENY")
+			w.WriteHeader(404)
+			return
+		}
+
+		if version := r.Header.Get("Gitlab-Workhorse"); version == "" {
+			log.Println("UPSTREAM", r.Method, r.URL, "DENY")
+			w.WriteHeader(403)
+			return
+		}
+
+		handler(w, r)
+	}))
+}
+
+func testAuthServer(url *regexp.Regexp, code int, body interface{}) *httptest.Server {
+	return testServerWithHandler(url, func(w http.ResponseWriter, r *http.Request) {
+		// Write pure string
+		if data, ok := body.(string); ok {
+			log.Println("UPSTREAM", r.Method, r.URL, code)
+			w.WriteHeader(code)
+			fmt.Fprint(w, data)
+			return
+		}
+
+		// Write json string
+		data, err := json.Marshal(body)
+		if err != nil {
+			log.Println("UPSTREAM", r.Method, r.URL, "FAILURE", err)
+			w.WriteHeader(503)
+			fmt.Fprint(w, err)
+			return
+		}
+
 		log.Println("UPSTREAM", r.Method, r.URL, code)
 		w.WriteHeader(code)
-		fmt.Fprint(w, body)
-	}))
+		w.Write(data)
+	})
 }
 
 func startWorkhorseServer(authBackend string) *httptest.Server {
@@ -301,23 +338,26 @@ func runOrFail(t *testing.T, cmd *exec.Cmd) {
 	}
 }
 
-func gitOkBody(t *testing.T) string {
-	return fmt.Sprintf(`{"GL_ID":"user-123","RepoPath":"%s"}`, repoPath(t))
+func gitOkBody(t *testing.T) interface{} {
+	return &authorizationResponse{
+		GL_ID:    "user-123",
+		RepoPath: repoPath(t),
+	}
 }
 
-func archiveOkBody(t *testing.T, archiveName string) string {
+func archiveOkBody(t *testing.T, archiveName string) interface{} {
 	cwd, err := os.Getwd()
 	if err != nil {
 		t.Fatal(err)
 	}
 	archivePath := path.Join(cwd, cacheDir, archiveName)
-	jsonString := `{
-		"RepoPath":"%s",
-		"ArchivePath":"%s",
-		"CommitId":"c7fbe50c7c7419d9701eebe64b1fdacc3df5b9dd",
-		"ArchivePrefix":"foobar123"
-	}`
-	return fmt.Sprintf(jsonString, repoPath(t), archivePath)
+
+	return &authorizationResponse{
+		RepoPath:      repoPath(t),
+		ArchivePath:   archivePath,
+		CommitId:      "c7fbe50c7c7419d9701eebe64b1fdacc3df5b9dd",
+		ArchivePrefix: "foobar123",
+	}
 }
 
 func repoPath(t *testing.T) string {

proxy.go

+package main
+
+import (
+	"net/http"
+)
+
+func proxyRequest(w http.ResponseWriter, r *gitRequest) {
+	upRequest, err := r.u.newUpstreamRequest(r.Request, r.Body, "")
+	if err != nil {
+		fail500(w, "newUpstreamRequest", err)
+		return
+	}
+
+	upResponse, err := r.u.httpClient.Do(upRequest)
+	if err != nil {
+		fail500(w, "do upstream request", err)
+		return
+	}
+	defer upResponse.Body.Close()
+
+	forwardResponseToClient(w, upResponse)
+}

proxy_test.go

+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"regexp"
+	"testing"
+)
+
+func TestProxyRequest(t *testing.T) {
+	ts := testServerWithHandler(regexp.MustCompile(`/url/path\z`), func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != "POST" {
+			t.Fatal("Expected POST request")
+		}
+
+		if r.Header.Get("Custom-Header") != "test" {
+			t.Fatal("Missing custom header")
+		}
+
+		var body bytes.Buffer
+		io.Copy(&body, r.Body)
+		if body.String() != "REQUEST" {
+			t.Fatal("Expected REQUEST in request body")
+		}
+
+		w.Header().Set("Custom-Response-Header", "test")
+		w.WriteHeader(202)
+		fmt.Fprint(w, "RESPONSE")
+	})
+
+	httpRequest, err := http.NewRequest("POST", ts.URL+"/url/path", bytes.NewBufferString("REQUEST"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	httpRequest.Header.Set("Custom-Header", "test")
+
+	request := gitRequest{
+		Request: httpRequest,
+		u:       newUpstream(ts.URL, nil),
+	}
+
+	response := httptest.NewRecorder()
+	proxyRequest(response, &request)
+	assertResponseCode(t, response, 202)
+
+	if response.Body.String() != "RESPONSE" {
+		t.Fatal("Expected RESPONSE in response body:", response.Body.String())
+	}
+
+	if response.Header().Get("Custom-Response-Header") != "test" {
+		t.Fatal("Expected custom response header")
+	}
+}

uploads.go

+package main
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"io/ioutil"
+	"mime/multipart"
+	"net/http"
+	"os"
+)
+
+func rewriteFormFilesFromMultipart(r *gitRequest, writer *multipart.Writer) (cleanup func(), err error) {
+	// Create multipart reader
+	reader, err := r.MultipartReader()
+	if err != nil {
+		return nil, err
+	}
+
+	var files []string
+
+	cleanup = func() {
+		for _, file := range files {
+			os.Remove(file)
+		}
+	}
+
+	// Execute cleanup in case of failure
+	defer func() {
+		if err != nil {
+			cleanup()
+		}
+	}()
+
+	for {
+		p, err := reader.NextPart()
+		if err == io.EOF {
+			break
+		}
+
+		name := p.FormName()
+		if name == "" {
+			continue
+		}
+
+		// Copy form field
+		if filename := p.FileName(); filename != "" {
+			// Create temporary directory where the uploaded file will be stored
+			if err := os.MkdirAll(r.TempPath, 0700); err != nil {
+				return cleanup, err
+			}
+
+			// Create temporary file in path returned by Authorization filter
+			file, err := ioutil.TempFile(r.TempPath, "upload_")
+			if err != nil {
+				return cleanup, err
+			}
+			defer file.Close()
+
+			// Add file entry
+			writer.WriteField(name+".path", file.Name())
+			writer.WriteField(name+".name", filename)
+			files = append(files, file.Name())
+
+			_, err = io.Copy(file, p)
+			file.Close()
+			if err != nil {
+				return cleanup, err
+			}
+		} else {
+			np, err := writer.CreatePart(p.Header)
+			if err != nil {
+				return cleanup, err
+			}
+
+			_, err = io.Copy(np, p)
+			if err != nil {
+				return cleanup, err
+			}
+		}
+	}
+	return cleanup, nil
+}
+
+func handleFileUploads(w http.ResponseWriter, r *gitRequest) {
+	if r.TempPath == "" {
+		fail500(w, "handleUploadFile", errors.New("missing temporary path"))
+		return
+	}
+
+	var body bytes.Buffer
+	writer := multipart.NewWriter(&body)
+	defer writer.Close()
+
+	// Rewrite multipart form data
+	cleanup, err := rewriteFormFilesFromMultipart(r, writer)
+	if err != nil {
+		if err == http.ErrNotMultipart {
+			proxyRequest(w, r)
+		} else {
+			fail500(w, "Couldn't handle upload request.", err)
+		}
+		return
+	}
+
+	if cleanup != nil {
+		defer cleanup()
+	}
+
+	// Close writer
+	writer.Close()
+
+	// Create request
+	upstreamRequest, err := r.u.newUpstreamRequest(r.Request, nil, "")
+	if err != nil {
+		fail500(w, "Couldn't handle artifacts upload request.", err)
+		return
+	}
+
+	// Set multipart form data
+	upstreamRequest.Body = ioutil.NopCloser(&body)
+	upstreamRequest.ContentLength = int64(body.Len())
+	upstreamRequest.Header.Set("Content-Type", writer.FormDataContentType())
+
+	// Forward request to backend
+	upstreamResponse, err := r.u.httpClient.Do(upstreamRequest)
+	if err != nil {
+		fail500(w, "do upstream request", err)
+		return
+	}
+	defer upstreamResponse.Body.Close()
+
+	forwardResponseToClient(w, upstreamResponse)
+}

uploads_test.go

+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"mime/multipart"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"regexp"
+	"strings"
+	"testing"
+)
+
+func TestUploadTempPathRequirement(t *testing.T) {
+	response := httptest.NewRecorder()
+	request := gitRequest{
+		authorizationResponse: authorizationResponse{
+			TempPath: "",
+		},
+	}
+	handleFileUploads(response, &request)
+	assertResponseCode(t, response, 500)
+}
+
+func TestUploadHandlerForwardingRawData(t *testing.T) {
+	ts := testServerWithHandler(regexp.MustCompile(`/url/path\z`), func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != "PATCH" {
+			t.Fatal("Expected PATCH request")
+		}
+
+		var body bytes.Buffer
+		io.Copy(&body, r.Body)
+		if body.String() != "REQUEST" {
+			t.Fatal("Expected REQUEST in request body")
+		}
+
+		w.WriteHeader(202)
+		fmt.Fprint(w, "RESPONSE")
+	})
+
+	httpRequest, err := http.NewRequest("PATCH", ts.URL+"/url/path", bytes.NewBufferString("REQUEST"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tempPath, err := ioutil.TempDir("", "uploads")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tempPath)
+
+	response := httptest.NewRecorder()
+	request := gitRequest{
+		Request: httpRequest,
+		u:       newUpstream(ts.URL, nil),
+		authorizationResponse: authorizationResponse{
+			TempPath: tempPath,
+		},
+	}
+	handleFileUploads(response, &request)
+	assertResponseCode(t, response, 202)
+	if response.Body.String() != "RESPONSE" {
+		t.Fatal("Expected RESPONSE in response body")
+	}
+}
+
+func TestUploadHandlerRewritingMultiPartData(t *testing.T) {
+	var filePath string
+
+	tempPath, err := ioutil.TempDir("", "uploads")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tempPath)
+
+	ts := testServerWithHandler(regexp.MustCompile(`/url/path\z`), func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != "PUT" {
+			t.Fatal("Expected PUT request")
+		}
+
+		err := r.ParseMultipartForm(100000)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if len(r.MultipartForm.Value) != 3 {
+			t.Fatal("Expected to receive exactly 3 values")
+		}
+
+		if len(r.MultipartForm.File) != 0 {
+			t.Fatal("Expected to not receive any files")
+		}
+
+		if r.FormValue("token") != "test" {
+			t.Fatal("Expected to receive token")
+		}
+
+		if r.FormValue("file.name") != "my.file" {
+			t.Fatal("Expected to receive a filename")
+		}
+
+		filePath = r.FormValue("file.path")
+
+		if !strings.HasPrefix(r.FormValue("file.path"), tempPath) {
+			t.Fatal("Expected to the file to be in tempPath")
+		}
+
+		w.WriteHeader(202)
+		fmt.Fprint(w, "RESPONSE")
+	})
+
+	var buffer bytes.Buffer
+
+	writer := multipart.NewWriter(&buffer)
+	writer.WriteField("token", "test")
+	file, err := writer.CreateFormFile("file", "my.file")
+	if err != nil {
+		t.Fatal(err)
+	}
+	fmt.Fprint(file, "test")
+	writer.Close()
+
+	httpRequest, err := http.NewRequest("PUT", ts.URL+"/url/path", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	httpRequest.Body = ioutil.NopCloser(&buffer)
+	httpRequest.ContentLength = int64(buffer.Len())
+	httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
+
+	response := httptest.NewRecorder()
+	request := gitRequest{
+		Request: httpRequest,
+		u:       newUpstream(ts.URL, nil),
+		authorizationResponse: authorizationResponse{
+			TempPath: tempPath,
+		},
+	}
+	handleFileUploads(response, &request)
+	assertResponseCode(t, response, 202)
+
+	if _, err := os.Stat(filePath); !os.IsNotExist(err) {
+		t.Fatal("expected the file to be deleted")
+	}
+}

upstream.go

@@ -51,6 +51,9 @@ type authorizationResponse struct {
 	LfsOid string
 	// LFS object size
 	LfsSize int64
+	// TmpPath is the path where we should store temporary files
+	// This is set by authorization middleware
+	TempPath string
 }
 
 // A gitReqest is an *http.Request decorated with attributes returned by the
@@ -64,16 +67,24 @@ type gitRequest struct {
 // Routing table
 var gitServices = [...]gitService{
 	gitService{"GET", regexp.MustCompile(`/info/refs\z`), repoPreAuthorizeHandler(handleGetInfoRefs)},
-	gitService{"POST", regexp.MustCompile(`/git-upload-pack\z`), repoPreAuthorizeHandler(handlePostRPC)},
-	gitService{"POST", regexp.MustCompile(`/git-receive-pack\z`), repoPreAuthorizeHandler(handlePostRPC)},
+	gitService{"POST", regexp.MustCompile(`/git-upload-pack\z`), repoPreAuthorizeHandler(contentEncodingHandler(handlePostRPC))},
+	gitService{"POST", regexp.MustCompile(`/git-receive-pack\z`), repoPreAuthorizeHandler(contentEncodingHandler(handlePostRPC))},
 	gitService{"GET", regexp.MustCompile(`/repository/archive\z`), repoPreAuthorizeHandler(handleGetArchive)},
 	gitService{"GET", regexp.MustCompile(`/repository/archive.zip\z`), repoPreAuthorizeHandler(handleGetArchive)},
 	gitService{"GET", regexp.MustCompile(`/repository/archive.tar\z`), repoPreAuthorizeHandler(handleGetArchive)},
 	gitService{"GET", regexp.MustCompile(`/repository/archive.tar.gz\z`), repoPreAuthorizeHandler(handleGetArchive)},
 	gitService{"GET", regexp.MustCompile(`/repository/archive.tar.bz2\z`), repoPreAuthorizeHandler(handleGetArchive)},
 	gitService{"GET", regexp.MustCompile(`/uploads/`), handleSendFile},
+
+	// Git LFS
 	gitService{"PUT", regexp.MustCompile(`/gitlab-lfs/objects/([0-9a-f]{64})/([0-9]+)\z`), lfsAuthorizeHandler(handleStoreLfsObject)},
 	gitService{"GET", regexp.MustCompile(`/gitlab-lfs/objects/([0-9a-f]{64})\z`), handleSendFile},
+
+	// CI artifacts
+	gitService{"GET", regexp.MustCompile(`/builds/download\z`), handleSendFile},
+	gitService{"GET", regexp.MustCompile(`/ci/api/v1/builds/[0-9]+/artifacts\z`), handleSendFile},
+	gitService{"POST", regexp.MustCompile(`/ci/api/v1/builds/[0-9]+/artifacts\z`), artifactsAuthorizeHandler(contentEncodingHandler(handleFileUploads))},
+	gitService{"DELETE", regexp.MustCompile(`/ci/api/v1/builds/[0-9]+/artifacts\z`), proxyRequest},
 }
 
 func newUpstream(authBackend string, authTransport http.RoundTripper) *upstream {
@@ -135,6 +146,7 @@ func (u *upstream) newUpstreamRequest(r *http.Request, body io.Reader, suffix st
 		authReq.Header.Del("Content-Type")
 		authReq.Header.Del("Content-Encoding")
 		authReq.Header.Del("Content-Length")
+		authReq.Header.Del("Content-Disposition")
 		authReq.Header.Del("Accept-Encoding")
 		authReq.Header.Del("Transfer-Encoding")
 	}