Implement artifacts upload mechanism

artifacts.go

+package main
+
+func artifactsAuthorizeHandler(handleFunc serviceHandleFunc) serviceHandleFunc {
+	return preAuthorizeHandler(handleFunc, "/authorize")
+}

git-http.go

@@ -5,7 +5,6 @@ In this file we handle the Git 'smart HTTP' protocol
 package main
 
 import (
-	"compress/gzip"
 	"fmt"
 	"io"
 	"net/http"
@@ -58,7 +57,6 @@ func handleGetInfoRefs(w http.ResponseWriter, r *gitRequest) {
 }
 
 func handlePostRPC(w http.ResponseWriter, r *gitRequest) {
-	var body io.ReadCloser
 	var err error
 
 	// Get Git action from URL
@@ -69,18 +67,6 @@ func handlePostRPC(w http.ResponseWriter, r *gitRequest) {
 		return
 	}
 
-	// The client request body may have been gzipped.
-	if r.Header.Get("Content-Encoding") == "gzip" {
-		body, err = gzip.NewReader(r.Body)
-		if err != nil {
-			fail500(w, "handlePostRPC", err)
-			return
-		}
-	} else {
-		body = r.Body
-	}
-	defer body.Close()
-
 	// Prepare our Git subprocess
 	cmd := gitCommand(r.GL_ID, "git", subCommand(action), "--stateless-rpc", r.RepoPath)
 	stdout, err := cmd.StdoutPipe()
@@ -102,7 +88,7 @@ func handlePostRPC(w http.ResponseWriter, r *gitRequest) {
 	defer cleanUpProcessGroup(cmd) // Ensure brute force subprocess clean-up
 
 	// Write the client request body to Git's standard input
-	if _, err := io.Copy(stdin, body); err != nil {
+	if _, err := io.Copy(stdin, r.Body); err != nil {
 		fail500(w, "handlePostRPC write to subprocess", err)
 		return
 	}
@@ -112,9 +98,6 @@ func handlePostRPC(w http.ResponseWriter, r *gitRequest) {
 	// It may take a while before we return and the deferred closes happen
 	// so let's free up some resources already.
 	r.Body.Close()
-	// If the body was compressed, body != r.Body and this frees up the
-	// gzip.Reader.
-	body.Close()
 
 	// Start writing the response
 	w.Header().Add("Content-Type", fmt.Sprintf("application/x-%s-result", action))

handlers.go

+package main
+
+import (
+	"compress/gzip"
+	"fmt"
+	"io"
+	"net/http"
+)
+
+func contentEncodingHandler(handleFunc serviceHandleFunc) serviceHandleFunc {
+	return func(w http.ResponseWriter, r *gitRequest) {
+		var body io.ReadCloser
+		var err error
+
+		// The client request body may have been gzipped.
+		contentEncoding := r.Header.Get("Content-Encoding")
+		switch contentEncoding {
+		case "":
+			body = r.Body
+		case "gzip":
+			body, err = gzip.NewReader(r.Body)
+		default:
+			err = fmt.Errorf("unsupported content encoding: %s", contentEncoding)
+		}
+
+		if err != nil {
+			fail500(w, "contentEncodingHandler", err)
+			return
+		}
+		defer body.Close()
+
+		r.Body = body
+		r.Header.Del("Content-Encoding")
+
+		handleFunc(w, r)
+	}
+}

helpers.go

@@ -6,13 +6,22 @@ package main
 
 import (
 	"fmt"
+	"io"
+	"io/ioutil"
 	"log"
 	"net/http"
+	"net/url"
 	"os"
 	"os/exec"
+	"strings"
 	"syscall"
 )
 
+func fail400(w http.ResponseWriter, context string, err error) {
+	http.Error(w, "Bad request", 400)
+	logContext(context, err)
+}
+
 func fail500(w http.ResponseWriter, context string, err error) {
 	http.Error(w, "Internal server error", 500)
 	logContext(context, err)
@@ -52,3 +61,21 @@ func cleanUpProcessGroup(cmd *exec.Cmd) {
 	// reap our child process
 	cmd.Wait()
 }
+
+func forwardResponseToClient(w http.ResponseWriter, r *http.Response) {
+	log.Printf("PROXY:%s %q %d", r.Request.Method, r.Request.URL, r.StatusCode)
+
+	for k, v := range r.Header {
+		w.Header()[k] = v
+	}
+
+	w.WriteHeader(r.StatusCode)
+	io.Copy(w, r.Body)
+}
+
+func setHttpPostForm(r *http.Request, values url.Values) {
+	dataBuffer := strings.NewReader(values.Encode())
+	r.Body = ioutil.NopCloser(dataBuffer)
+	r.ContentLength = int64(dataBuffer.Len())
+	r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+}

proxy.go

+package main
+
+import (
+	"net/http"
+)
+
+func proxyRequest(w http.ResponseWriter, r *gitRequest) {
+	upRequest, err := r.u.newUpstreamRequest(r.Request, r.Body, "")
+	if err != nil {
+		fail500(w, "newUpstreamRequest", err)
+		return
+	}
+
+	upResponse, err := r.u.httpClient.Do(upRequest)
+	if err != nil {
+		fail500(w, "do upstream request", err)
+		return
+	}
+	defer upResponse.Body.Close()
+
+	forwardResponseToClient(w, upResponse)
+}

uploads.go

+package main
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"io/ioutil"
+	"mime/multipart"
+	"net/http"
+	"os"
+)
+
+func rewriteFormFilesFromMultipart(r *gitRequest, writer *multipart.Writer) (cleanup func(), err error) {
+	// Create multipart reader
+	reader, err := r.MultipartReader()
+	if err != nil {
+		return nil, err
+	}
+
+	var files []string
+
+	cleanup = func() {
+		for _, file := range files {
+			os.Remove(file)
+		}
+	}
+
+	// Execute cleanup in case of failure
+	defer func() {
+		if err != nil {
+			cleanup()
+		}
+	}()
+
+	for {
+		p, err := reader.NextPart()
+		if err == io.EOF {
+			break
+		}
+
+		name := p.FormName()
+		if name == "" {
+			continue
+		}
+
+		// Copy form field
+		if filename := p.FileName(); filename != "" {
+			// Create temporary directory where the uploaded file will be stored
+			if err := os.MkdirAll(r.TempPath, 0700); err != nil {
+				return cleanup, err
+			}
+
+			// Create temporary file in path returned by Authorization filter
+			file, err := ioutil.TempFile(r.TempPath, "upload_")
+			if err != nil {
+				return cleanup, err
+			}
+			defer file.Close()
+
+			// Add file entry
+			writer.WriteField(name+".path", file.Name())
+			writer.WriteField(name+".file", filename)
+			files = append(files, file.Name())
+
+			_, err = io.Copy(file, p)
+			file.Close()
+			if err != nil {
+				return cleanup, err
+			}
+		} else {
+			np, err := writer.CreatePart(p.Header)
+			if err != nil {
+				return cleanup, err
+			}
+
+			_, err = io.Copy(np, p)
+			if err != nil {
+				return cleanup, err
+			}
+		}
+	}
+	return cleanup, nil
+}
+
+func handleFileUploads(w http.ResponseWriter, r *gitRequest) {
+	if r.TempPath == "" {
+		fail500(w, "handleUploadFile", errors.New("missing temporary path"))
+		return
+	}
+
+	var body bytes.Buffer
+	writer := multipart.NewWriter(&body)
+	defer writer.Close()
+
+	// Rewrite multipart form data
+	cleanup, err := rewriteFormFilesFromMultipart(r, writer)
+	if err != nil {
+		if err == http.ErrNotMultipart {
+			proxyRequest(w, r)
+		} else {
+			fail500(w, "Couldn't handle upload request.", err)
+		}
+		return
+	}
+
+	if cleanup != nil {
+		defer cleanup()
+	}
+
+	// Close writer
+	writer.Close()
+
+	// Create request
+	upstreamRequest, err := r.u.newUpstreamRequest(r.Request, nil, "")
+	if err != nil {
+		fail500(w, "Couldn't handle artifacts upload request.", err)
+		return
+	}
+
+	// Set multipart form data
+	upstreamRequest.Body = ioutil.NopCloser(&body)
+	upstreamRequest.ContentLength = int64(body.Len())
+	upstreamRequest.Header.Set("Content-Type", writer.FormDataContentType())
+
+	// Forward request to backend
+	upstreamResponse, err := r.u.httpClient.Do(upstreamRequest)
+	if err != nil {
+		fail500(w, "do upstream request", err)
+		return
+	}
+	defer upstreamResponse.Body.Close()
+
+	forwardResponseToClient(w, upstreamResponse)
+}

upstream.go

@@ -51,6 +51,9 @@ type authorizationResponse struct {
 	LfsOid string
 	// LFS object size
 	LfsSize int64
+	// TmpPath is the path where we should store temporary files
+	// This is set by authorization middleware
+	TempPath string
 }
 
 // A gitReqest is an *http.Request decorated with attributes returned by the
@@ -64,16 +67,24 @@ type gitRequest struct {
 // Routing table
 var gitServices = [...]gitService{
 	gitService{"GET", regexp.MustCompile(`/info/refs\z`), repoPreAuthorizeHandler(handleGetInfoRefs)},
-	gitService{"POST", regexp.MustCompile(`/git-upload-pack\z`), repoPreAuthorizeHandler(handlePostRPC)},
-	gitService{"POST", regexp.MustCompile(`/git-receive-pack\z`), repoPreAuthorizeHandler(handlePostRPC)},
+	gitService{"POST", regexp.MustCompile(`/git-upload-pack\z`), repoPreAuthorizeHandler(contentEncodingHandler(handlePostRPC))},
+	gitService{"POST", regexp.MustCompile(`/git-receive-pack\z`), repoPreAuthorizeHandler(contentEncodingHandler(handlePostRPC))},
 	gitService{"GET", regexp.MustCompile(`/repository/archive\z`), repoPreAuthorizeHandler(handleGetArchive)},
 	gitService{"GET", regexp.MustCompile(`/repository/archive.zip\z`), repoPreAuthorizeHandler(handleGetArchive)},
 	gitService{"GET", regexp.MustCompile(`/repository/archive.tar\z`), repoPreAuthorizeHandler(handleGetArchive)},
 	gitService{"GET", regexp.MustCompile(`/repository/archive.tar.gz\z`), repoPreAuthorizeHandler(handleGetArchive)},
 	gitService{"GET", regexp.MustCompile(`/repository/archive.tar.bz2\z`), repoPreAuthorizeHandler(handleGetArchive)},
 	gitService{"GET", regexp.MustCompile(`/uploads/`), handleSendFile},
+
+	// Git LFS
 	gitService{"PUT", regexp.MustCompile(`/gitlab-lfs/objects/([0-9a-f]{64})/([0-9]+)\z`), lfsAuthorizeHandler(handleStoreLfsObject)},
 	gitService{"GET", regexp.MustCompile(`/gitlab-lfs/objects/([0-9a-f]{64})\z`), handleSendFile},
+
+	// CI artifacts
+	gitService{"GET", regexp.MustCompile(`/builds/download\z`), handleSendFile},
+	gitService{"GET", regexp.MustCompile(`/ci/api/v1/builds/[0-9]+/artifacts\z`), handleSendFile},
+	gitService{"POST", regexp.MustCompile(`/ci/api/v1/builds/[0-9]+/artifacts\z`), artifactsAuthorizeHandler(contentEncodingHandler(handleFileUploads))},
+	gitService{"DELETE", regexp.MustCompile(`/ci/api/v1/builds/[0-9]+/artifacts\z`), proxyRequest},
 }
 
 func newUpstream(authBackend string, authTransport http.RoundTripper) *upstream {
@@ -135,6 +146,7 @@ func (u *upstream) newUpstreamRequest(r *http.Request, body io.Reader, suffix st
 		authReq.Header.Del("Content-Type")
 		authReq.Header.Del("Content-Encoding")
 		authReq.Header.Del("Content-Length")
+		authReq.Header.Del("Content-Disposition")
 		authReq.Header.Del("Accept-Encoding")
 		authReq.Header.Del("Transfer-Encoding")
 	}