Use gitlab-zip-cat to send zip entries

.gitignore

@@ -2,3 +2,4 @@ gitlab-workhorse
 testdata/data
 testdata/scratch
 testdata/public
+gitlab-zip-cat

Makefile

 PREFIX=/usr/local
 VERSION=$(shell git describe)-$(shell date -u +%Y%m%d.%H%M%S)
+GOBUILD=go build -ldflags "-X main.Version=${VERSION}"
+
+all: gitlab-zip-cat gitlab-workhorse
+
+gitlab-zip-cat:	$(shell find cmd/gitlab-zip-cat/ -name '*.go')
+	${GOBUILD} -o $@ ./cmd/$@
 	
 gitlab-workhorse: $(shell find . -name '*.go')
-	go build -ldflags "-X main.Version=${VERSION}" -o gitlab-workhorse
+	${GOBUILD} -o $@
 
-install: gitlab-workhorse
-	install gitlab-workhorse ${PREFIX}/bin/
+install: gitlab-workhorse gitlab-zip-cat
+	install gitlab-workhorse gitlab-zip-cat ${PREFIX}/bin/
 
 .PHONY: test
-test: testdata/data/group/test.git clean-workhorse gitlab-workhorse
+test: testdata/data/group/test.git clean-workhorse gitlab-workhorse gitlab-zip-cat
 	go fmt ./... | awk '{ print } END { if (NR > 0) { print "Please run go fmt"; exit 1 } }'
-	go test ./...
+	support/path-add-current go test ./...
 	@echo SUCCESS
 
 coverage: testdata/data/group/test.git
@@ -30,4 +36,4 @@ clean:	clean-workhorse
 
 .PHONY:	clean-workhorse
 clean-workhorse:
-	rm -f gitlab-workhorse
+	rm -f gitlab-workhorse gitlab-zip-cat

cmd/gitlab-zip-cat/main.go

+package main
+
+import (
+	"archive/zip"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+)
+
+const notFound = 2
+const progName = "gitlab-zip-cat"
+
+var Version = "unknown"
+
+var printVersion = flag.Bool("version", false, "Print version and exit")
+
+func main() {
+	flag.Parse()
+
+	version := fmt.Sprintf("%s %s", progName, Version)
+	if *printVersion {
+		fmt.Println(version)
+		os.Exit(0)
+	}
+
+	archiveFileName := os.Args[1]
+	fileName := os.Args[2]
+	archive, err := zip.OpenReader(archiveFileName)
+	if err != nil {
+		printError(fmt.Errorf("open %q: %v", archiveFileName, err))
+		os.Exit(notFound)
+	}
+	defer archive.Close()
+
+	file := findFileInZip(fileName, &archive.Reader)
+	if file == nil {
+		printError(fmt.Errorf("find %q in %q: not found", fileName, archiveFileName))
+		os.Exit(notFound)
+	}
+	// Start decompressing the file
+	reader, err := file.Open()
+	if err != nil {
+		fatalError(fmt.Errorf("open %q in %q: %v", fileName, archiveFileName, err))
+	}
+	defer reader.Close()
+	if _, err := io.Copy(os.Stdout, reader); err != nil {
+		fatalError(fmt.Errorf("write %q from %q to stdout: %v", fileName, archiveFileName, err))
+	}
+
+}
+
+func findFileInZip(fileName string, archive *zip.Reader) *zip.File {
+	for _, file := range archive.File {
+		if file.Name == fileName {
+			return file
+		}
+	}
+	return nil
+}
+
+func printError(err error) {
+	log.Printf("%s: %v", progName, err)
+}
+
+func fatalError(err error) {
+	printError(err)
+	os.Exit(1)
+}

internal/artifacts/artifact_download.go

@@ -3,7 +3,6 @@ package artifacts
 import (
 	"../api"
 	"../helper"
-	"archive/zip"
 	"encoding/base64"
 	"errors"
 	"fmt"
@@ -11,10 +10,13 @@ import (
 	"mime"
 	"net/http"
 	"os"
+	"os/exec"
 	"path/filepath"
-	"strconv"
+	"syscall"
 )
 
+const exitStatusNotFound = 2
+
 func decodeFileEntry(entry string) (string, error) {
 	decoded, err := base64.StdEncoding.DecodeString(entry)
 	if err != nil {
@@ -31,44 +33,38 @@ func detectFileContentType(fileName string) string {
 	return contentType
 }
 
-func findFileInZip(fileName string, archive *zip.Reader) *zip.File {
-	for _, file := range archive.File {
-		if file.Name == fileName {
-			return file
-		}
-	}
-	return nil
-}
-
 func unpackFileFromZip(archiveFileName, fileName string, headers http.Header, output io.Writer) error {
-	archive, err := zip.OpenReader(archiveFileName)
+	catFile := exec.Command("gitlab-zip-cat", archiveFileName, fileName)
+	catFile.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+	stdout, err := catFile.StdoutPipe()
 	if err != nil {
-		return err
+		return fmt.Errorf("create gitlab-zip-cat stdout pipe: %v", err)
 	}
-	defer archive.Close()
 
-	file := findFileInZip(fileName, &archive.Reader)
-	if file == nil {
-		return os.ErrNotExist
-	}
-
-	// Start decompressing the file
-	reader, err := file.Open()
-	if err != nil {
-		return err
+	if err := catFile.Start(); err != nil {
+		return fmt.Errorf("start %v: %v", catFile.Args, err)
 	}
-	defer reader.Close()
-
+	defer helper.CleanUpProcessGroup(catFile)
 	basename := filepath.Base(fileName)
 
 	// Write http headers about the file
-	headers.Set("Content-Length", strconv.FormatInt(int64(file.UncompressedSize64), 10))
-	headers.Set("Content-Type", detectFileContentType(file.Name))
+	headers.Set("Content-Type", detectFileContentType(fileName))
 	headers.Set("Content-Disposition", "attachment; filename=\""+escapeQuotes(basename)+"\"")
 
 	// Copy file body to client
-	_, err = io.Copy(output, reader)
-	return err
+	if _, err := io.Copy(output, stdout); err != nil {
+		return fmt.Errorf("copy %v stdout: %v", catFile.Args, err)
+	}
+
+	if err := catFile.Wait(); err != nil {
+		if st, ok := helper.ExitStatus(err); ok && st == exitStatusNotFound {
+			return os.ErrNotExist
+		}
+
+		return fmt.Errorf("wait for %v to finish: %v", catFile.Args, err)
+	}
+
+	return nil
 }
 
 // Artifacts downloader doesn't support ranges when downloading a single file

internal/git/archive.go

@@ -79,7 +79,7 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
 		helper.Fail500(w, fmt.Errorf("handleGetArchive: start %v: %v", archiveCmd.Args, err))
 		return
 	}
-	defer cleanUpProcessGroup(archiveCmd) // Ensure brute force subprocess clean-up
+	defer helper.CleanUpProcessGroup(archiveCmd) // Ensure brute force subprocess clean-up
 
 	var stdout io.ReadCloser
 	if compressCmd == nil {
@@ -99,7 +99,7 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
 			helper.Fail500(w, fmt.Errorf("handleGetArchive: start %v: %v", compressCmd.Args, err))
 			return
 		}
-		defer cleanUpProcessGroup(compressCmd)
+		defer helper.CleanUpProcessGroup(compressCmd)
 
 		archiveStdout.Close()
 	}

internal/git/command.go

@@ -23,18 +23,3 @@ func gitCommand(gl_id string, name string, args ...string) *exec.Cmd {
 	cmd.Stderr = os.Stderr
 	return cmd
 }
-
-func cleanUpProcessGroup(cmd *exec.Cmd) {
-	if cmd == nil {
-		return
-	}
-
-	process := cmd.Process
-	if process != nil && process.Pid > 0 {
-		// Send SIGTERM to the process group of cmd
-		syscall.Kill(-process.Pid, syscall.SIGTERM)
-	}
-
-	// reap our child process
-	cmd.Wait()
-}

internal/git/git-http.go

@@ -72,7 +72,7 @@ func handleGetInfoRefs(w http.ResponseWriter, r *http.Request, a *api.Response)
 		helper.Fail500(w, fmt.Errorf("handleGetInfoRefs: start %v: %v", cmd.Args, err))
 		return
 	}
-	defer cleanUpProcessGroup(cmd) // Ensure brute force subprocess clean-up
+	defer helper.CleanUpProcessGroup(cmd) // Ensure brute force subprocess clean-up
 
 	// Start writing the response
 	w.Header().Add("Content-Type", fmt.Sprintf("application/x-%s-advertisement", rpc))
@@ -125,7 +125,7 @@ func handlePostRPC(w http.ResponseWriter, r *http.Request, a *api.Response) {
 		helper.Fail500(w, fmt.Errorf("handlePostRPC: start %v: %v", cmd.Args, err))
 		return
 	}
-	defer cleanUpProcessGroup(cmd) // Ensure brute force subprocess clean-up
+	defer helper.CleanUpProcessGroup(cmd) // Ensure brute force subprocess clean-up
 
 	// Write the client request body to Git's standard input
 	if _, err := io.Copy(stdin, r.Body); err != nil {

internal/helper/helpers.go

@@ -6,6 +6,8 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"os/exec"
+	"syscall"
 )
 
 func Fail500(w http.ResponseWriter, err error) {
@@ -69,3 +71,32 @@ func HTTPError(w http.ResponseWriter, r *http.Request, error string, code int) {
 
 	http.Error(w, error, code)
 }
+
+func CleanUpProcessGroup(cmd *exec.Cmd) {
+	if cmd == nil {
+		return
+	}
+
+	process := cmd.Process
+	if process != nil && process.Pid > 0 {
+		// Send SIGTERM to the process group of cmd
+		syscall.Kill(-process.Pid, syscall.SIGTERM)
+	}
+
+	// reap our child process
+	cmd.Wait()
+}
+
+func ExitStatus(err error) (int, bool) {
+	exitError, ok := err.(*exec.ExitError)
+	if !ok {
+		return 0, false
+	}
+
+	waitStatus, ok := exitError.Sys().(syscall.WaitStatus)
+	if !ok {
+		return 0, false
+	}
+
+	return waitStatus.ExitStatus(), true
+}

support/path-add-current

+#!/bin/sh
+exec env PATH=$(pwd):${PATH} "$@"