Merge branch 'master' of gitlab.com:gitlab-org/gitlab-workhorse

.gitignore

-gitlab-git-http-server
 test/data
 test/scratch
+gitlab-workhorse

CHANGELOG

-# Changelog for gitlab-git-http-server
+# Changelog for gitlab-workhorse
+
+Formerly known as 'gitlab-git-http-server'.
+
+0.4.0
+
+Rename the project to gitlab-workhorse. The old name had become too
+specific.
+
+Other changes:
+
+- pass LD_LIBRARY_PATH to Git commands
+- accomodate broken HTTP clients by spelling 'Www-Authenticate' as
+  'WWW-Authenticate'
 
 0.3.1
 

Makefile

 PREFIX=/usr/local
 VERSION=$(shell git describe)-$(shell date -u +%Y%m%d.%H%M%S)
 
-gitlab-git-http-server: main.go githandler.go
-	go build -ldflags "-X main.Version ${VERSION}" -o gitlab-git-http-server
+gitlab-workhorse: main.go githandler.go archive.go git-http.go helpers.go
+	go build -ldflags "-X main.Version ${VERSION}" -o gitlab-workhorse
 
-install: gitlab-git-http-server
-	install gitlab-git-http-server ${PREFIX}/bin/
+install: gitlab-workhorse
+	install gitlab-workhorse ${PREFIX}/bin/
 
 .PHONY: test
 test: test/data/test.git
@@ -19,5 +19,5 @@ test/data:
 
 .PHONY: clean
 clean:
-	rm -f gitlab-git-http-server
+	rm -f gitlab-workhorse
 	rm -rf test/data test/scratch

README.md

-# gitlab-git-http-server
+# gitlab-workhorse
 
-## Renaming to gitlab-workhorse
-
-Starting with GitLab 8.2, this project has been renamed to
-gitlab-workhorse. The new URL is
-https://gitlab.com/gitlab-org/gitlab-workhorse . The code in this
-repository will no longer be updated.
-
-## Original preamble
-
-gitlab-git-http-server was designed to unload Git HTTP traffic from
+gitlab-workhorse was designed to unload Git HTTP traffic from
 the GitLab Rails app (Unicorn) to a separate daemon.  It also serves
 'git archive' downloads for GitLab.  All authentication and
 authorization logic is still handled by the GitLab Rails app.
 
-Architecture: Git client -> NGINX -> gitlab-git-http-server (makes
+Architecture: Git client -> NGINX -> gitlab-workhorse (makes
 auth request to GitLab Rails app) -> git-upload-pack
 
 ## Usage
 
 ```
-  gitlab-git-http-server [OPTIONS]
+  gitlab-workhorse [OPTIONS]
 
 Options:
   -authBackend string
@@ -39,15 +30,15 @@ Options:
     	Print version and exit
 ```
 
-gitlab-git-http-server allows Git HTTP clients to push and pull to
+gitlab-workhorse allows Git HTTP clients to push and pull to
 and from Git repositories. Each incoming request is first replayed
 (with an empty request body) to an external authentication/authorization
 HTTP server: the 'auth backend'. The auth backend is expected to
 be a GitLab Unicorn process.  The 'auth response' is a JSON message
-which tells gitlab-git-http-server the path of the Git repository
+which tells gitlab-workhorse the path of the Git repository
 to read from/write to.
 
-gitlab-git-http-server can listen on either a TCP or a Unix domain socket. It
+gitlab-workhorse can listen on either a TCP or a Unix domain socket. It
 can also open a second listening TCP listening socket with the Go
 [net/http/pprof profiler server](http://golang.org/pkg/net/http/pprof/).
 
@@ -79,9 +70,9 @@ You can try out the Git server without authentication as follows:
 # Start a fake auth backend that allows everything/everybody
 make test/data/test.git
 go run support/fake-auth-backend.go ~+/test/data/test.git &
-# Start gitlab-git-http-server
+# Start gitlab-workhorse
 make
-./gitlab-git-http-server
+./gitlab-workhorse
 ```
 
 Now you can try things like:
@@ -94,14 +85,14 @@ curl -JO http://localhost:8181/test/repository/archive.zip
 ## Example request flow
 
 - start POST repo.git/git-receive-pack to NGINX
-- ..start POST repo.git/git-receive-pack to gitlab-git-http-server
+- ..start POST repo.git/git-receive-pack to gitlab-workhorse
 - ....start POST repo.git/git-receive-pack to Unicorn for auth
 - ....end POST to Unicorn for auth
-- ....start git-receive-pack process from gitlab-git-http-server
+- ....start git-receive-pack process from gitlab-workhorse
 - ......start POST /api/v3/internal/allowed to Unicorn from Git hook (check protected branches)
 - ......end POST to Unicorn from Git hook
 - ....end git-receive-pack process
-- ..end POST to gitlab-git-http-server
+- ..end POST to gitlab-workhorse
 - end POST to NGINX
 
 ## License

VERSION

-0.3.1
+0.4.0

archive.go

+/*
+In this file we handle 'git archive' downloads
+*/
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"os/exec"
+	"path"
+	"time"
+)
+
+func handleGetArchive(w http.ResponseWriter, r *gitRequest, format string) {
+	archiveFilename := path.Base(r.ArchivePath)
+
+	if cachedArchive, err := os.Open(r.ArchivePath); err == nil {
+		defer cachedArchive.Close()
+		log.Printf("Serving cached file %q", r.ArchivePath)
+		setArchiveHeaders(w, format, archiveFilename)
+		// Even if somebody deleted the cachedArchive from disk since we opened
+		// the file, Unix file semantics guarantee we can still read from the
+		// open file in this process.
+		http.ServeContent(w, r.Request, "", time.Unix(0, 0), cachedArchive)
+		return
+	}
+
+	// We assume the tempFile has a unique name so that concurrent requests are
+	// safe. We create the tempfile in the same directory as the final cached
+	// archive we want to create so that we can use an atomic link(2) operation
+	// to finalize the cached archive.
+	tempFile, err := prepareArchiveTempfile(path.Dir(r.ArchivePath), archiveFilename)
+	if err != nil {
+		fail500(w, "handleGetArchive create tempfile for archive", err)
+	}
+	defer tempFile.Close()
+	defer os.Remove(tempFile.Name())
+
+	compressCmd, archiveFormat := parseArchiveFormat(format)
+
+	archiveCmd := gitCommand("", "git", "--git-dir="+r.RepoPath, "archive", "--format="+archiveFormat, "--prefix="+r.ArchivePrefix+"/", r.CommitId)
+	archiveStdout, err := archiveCmd.StdoutPipe()
+	if err != nil {
+		fail500(w, "handleGetArchive", err)
+		return
+	}
+	defer archiveStdout.Close()
+	if err := archiveCmd.Start(); err != nil {
+		fail500(w, "handleGetArchive", err)
+		return
+	}
+	defer cleanUpProcessGroup(archiveCmd) // Ensure brute force subprocess clean-up
+
+	var stdout io.ReadCloser
+	if compressCmd == nil {
+		stdout = archiveStdout
+	} else {
+		compressCmd.Stdin = archiveStdout
+
+		stdout, err = compressCmd.StdoutPipe()
+		if err != nil {
+			fail500(w, "handleGetArchive compressCmd stdout pipe", err)
+			return
+		}
+		defer stdout.Close()
+
+		if err := compressCmd.Start(); err != nil {
+			fail500(w, "handleGetArchive start compressCmd process", err)
+			return
+		}
+		defer compressCmd.Wait()
+
+		archiveStdout.Close()
+	}
+	// Every Read() from stdout will be synchronously written to tempFile
+	// before it comes out the TeeReader.
+	archiveReader := io.TeeReader(stdout, tempFile)
+
+	// Start writing the response
+	setArchiveHeaders(w, format, archiveFilename)
+	w.WriteHeader(200) // Don't bother with HTTP 500 from this point on, just return
+	if _, err := io.Copy(w, archiveReader); err != nil {
+		logContext("handleGetArchive read from subprocess", err)
+		return
+	}
+	if err := archiveCmd.Wait(); err != nil {
+		logContext("handleGetArchive wait for archiveCmd", err)
+		return
+	}
+	if compressCmd != nil {
+		if err := compressCmd.Wait(); err != nil {
+			logContext("handleGetArchive wait for compressCmd", err)
+			return
+		}
+	}
+
+	if err := finalizeCachedArchive(tempFile, r.ArchivePath); err != nil {
+		logContext("handleGetArchive finalize cached archive", err)
+		return
+	}
+}
+
+func setArchiveHeaders(w http.ResponseWriter, format string, archiveFilename string) {
+	w.Header().Add("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, archiveFilename))
+	if format == "zip" {
+		w.Header().Add("Content-Type", "application/zip")
+	} else {
+		w.Header().Add("Content-Type", "application/octet-stream")
+	}
+	w.Header().Add("Content-Transfer-Encoding", "binary")
+	w.Header().Add("Cache-Control", "private")
+}
+
+func parseArchiveFormat(format string) (*exec.Cmd, string) {
+	switch format {
+	case "tar":
+		return nil, "tar"
+	case "tar.gz":
+		return exec.Command("gzip", "-c", "-n"), "tar"
+	case "tar.bz2":
+		return exec.Command("bzip2", "-c"), "tar"
+	case "zip":
+		return nil, "zip"
+	}
+	return nil, "unknown"
+}
+
+func prepareArchiveTempfile(dir string, prefix string) (*os.File, error) {
+	if err := os.MkdirAll(dir, 0700); err != nil {
+		return nil, err
+	}
+	return ioutil.TempFile(dir, prefix)
+}
+
+func finalizeCachedArchive(tempFile *os.File, archivePath string) error {
+	if err := tempFile.Close(); err != nil {
+		return err
+	}
+	return os.Link(tempFile.Name(), archivePath)
+}

git-http.go

+/*
+In this file we handle the Git 'smart HTTP' protocol
+*/
+
+package main
+
+import (
+	"compress/gzip"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+)
+
+func handleGetInfoRefs(w http.ResponseWriter, r *gitRequest, _ string) {
+	rpc := r.URL.Query().Get("service")
+	if !(rpc == "git-upload-pack" || rpc == "git-receive-pack") {
+		// The 'dumb' Git HTTP protocol is not supported
+		http.Error(w, "Not Found", 404)
+		return
+	}
+
+	// Prepare our Git subprocess
+	cmd := gitCommand(r.GL_ID, "git", subCommand(rpc), "--stateless-rpc", "--advertise-refs", r.RepoPath)
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		fail500(w, "handleGetInfoRefs", err)
+		return
+	}
+	defer stdout.Close()
+	if err := cmd.Start(); err != nil {
+		fail500(w, "handleGetInfoRefs", err)
+		return
+	}
+	defer cleanUpProcessGroup(cmd) // Ensure brute force subprocess clean-up
+
+	// Start writing the response
+	w.Header().Add("Content-Type", fmt.Sprintf("application/x-%s-advertisement", rpc))
+	w.Header().Add("Cache-Control", "no-cache")
+	w.WriteHeader(200) // Don't bother with HTTP 500 from this point on, just return
+	if err := pktLine(w, fmt.Sprintf("# service=%s\n", rpc)); err != nil {
+		logContext("handleGetInfoRefs response", err)
+		return
+	}
+	if err := pktFlush(w); err != nil {
+		logContext("handleGetInfoRefs response", err)
+		return
+	}
+	if _, err := io.Copy(w, stdout); err != nil {
+		logContext("handleGetInfoRefs read from subprocess", err)
+		return
+	}
+	if err := cmd.Wait(); err != nil {
+		logContext("handleGetInfoRefs wait for subprocess", err)
+		return
+	}
+}
+
+func handlePostRPC(w http.ResponseWriter, r *gitRequest, rpc string) {
+	var body io.ReadCloser
+	var err error
+
+	// The client request body may have been gzipped.
+	if r.Header.Get("Content-Encoding") == "gzip" {
+		body, err = gzip.NewReader(r.Body)
+		if err != nil {
+			fail500(w, "handlePostRPC", err)
+			return
+		}
+	} else {
+		body = r.Body
+	}
+	defer body.Close()
+
+	// Prepare our Git subprocess
+	cmd := gitCommand(r.GL_ID, "git", subCommand(rpc), "--stateless-rpc", r.RepoPath)
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		fail500(w, "handlePostRPC", err)
+		return
+	}
+	defer stdout.Close()
+	stdin, err := cmd.StdinPipe()
+	if err != nil {
+		fail500(w, "handlePostRPC", err)
+		return
+	}
+	defer stdin.Close()
+	if err := cmd.Start(); err != nil {
+		fail500(w, "handlePostRPC", err)
+		return
+	}
+	defer cleanUpProcessGroup(cmd) // Ensure brute force subprocess clean-up
+
+	// Write the client request body to Git's standard input
+	if _, err := io.Copy(stdin, body); err != nil {
+		fail500(w, "handlePostRPC write to subprocess", err)
+		return
+	}
+	// Signal to the Git subprocess that no more data is coming
+	stdin.Close()
+
+	// It may take a while before we return and the deferred closes happen
+	// so let's free up some resources already.
+	r.Body.Close()
+	// If the body was compressed, body != r.Body and this frees up the
+	// gzip.Reader.
+	body.Close()
+
+	// Start writing the response
+	w.Header().Add("Content-Type", fmt.Sprintf("application/x-%s-result", rpc))
+	w.Header().Add("Cache-Control", "no-cache")
+	w.WriteHeader(200) // Don't bother with HTTP 500 from this point on, just return
+
+	// This io.Copy may take a long time, both for Git push and pull.
+	if _, err := io.Copy(w, stdout); err != nil {
+		logContext("handlePostRPC read from subprocess", err)
+		return
+	}
+	if err := cmd.Wait(); err != nil {
+		logContext("handlePostRPC wait for subprocess", err)
+		return
+	}
+}
+
+func subCommand(rpc string) string {
+	return strings.TrimPrefix(rpc, "git-")
+}
+
+func pktLine(w io.Writer, s string) error {
+	_, err := fmt.Fprintf(w, "%04x%s", len(s)+4, s)
+	return err
+}
+
+func pktFlush(w io.Writer) error {
+	_, err := fmt.Fprint(w, "0000")
+	return err
+}

helpers.go

+/*
+Miscellaneous helpers: logging, errors, subprocesses
+*/
+
+package main
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"os/exec"
+	"syscall"
+)
+
+func fail500(w http.ResponseWriter, context string, err error) {
+	http.Error(w, "Internal server error", 500)
+	logContext(context, err)
+}
+
+func logContext(context string, err error) {
+	log.Printf("%s: %v", context, err)
+}
+
+// Git subprocess helpers
+func gitCommand(gl_id string, name string, args ...string) *exec.Cmd {
+	cmd := exec.Command(name, args...)
+	// Start the command in its own process group (nice for signalling)
+	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+	// Explicitly set the environment for the Git command
+	cmd.Env = []string{
+		fmt.Sprintf("PATH=%s", os.Getenv("PATH")),
+		fmt.Sprintf("LD_LIBRARY_PATH=%s", os.Getenv("LD_LIBRARY_PATH")),
+		fmt.Sprintf("GL_ID=%s", gl_id),
+	}
+	// If we don't do something with cmd.Stderr, Git errors will be lost
+	cmd.Stderr = os.Stderr
+	return cmd
+}
+
+func cleanUpProcessGroup(cmd *exec.Cmd) {
+	if cmd == nil {
+		return
+	}
+
+	process := cmd.Process
+	if process != nil && process.Pid > 0 {
+		// Send SIGTERM to the process group of cmd
+		syscall.Kill(-process.Pid, syscall.SIGTERM)
+	}
+
+	// reap our child process
+	cmd.Wait()
+}

main.go

 /*
-gitlab-git-http-server handles 'smart' Git HTTP requests for GitLab
+gitlab-workhorse handles slow requests for GitLab
 
 This HTTP server can service 'git clone', 'git push' etc. commands
 from Git clients that use the 'smart' Git HTTP protocol (git-upload-pack
@@ -9,8 +9,7 @@ backend (for authentication and authorization) and local disk access
 to Git repositories managed by GitLab. In GitLab, this role was previously
 performed by gitlab-grack.
 
-This file contains the main() function. Actual Git HTTP requests are handled by
-the gitHandler type, implemented in githandler.go.
+In this file we start the web server and hand off to the gitHandler type.
 */
 package main
 
@@ -43,7 +42,7 @@ func main() {
 	}
 	flag.Parse()
 
-	version := fmt.Sprintf("gitlab-git-http-server %s", Version)
+	version := fmt.Sprintf("gitlab-workhorse %s", Version)
 	if *printVersion {
 		fmt.Println(version)
 		os.Exit(0)

main_test.go

@@ -269,7 +269,7 @@ func testAuthServer(code int, body string) *httptest.Server {
 }
 
 func startServerOrFail(t *testing.T, ts *httptest.Server) *exec.Cmd {
-	cmd := exec.Command("go", "run", "main.go", "githandler.go", fmt.Sprintf("-authBackend=%s", ts.URL), fmt.Sprintf("-listenAddr=%s", servAddr))
+	cmd := exec.Command("go", "run", "main.go", "githandler.go", "archive.go", "git-http.go", "helpers.go", fmt.Sprintf("-authBackend=%s", ts.URL), fmt.Sprintf("-listenAddr=%s", servAddr))
 	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr