Commit 1ccba44f authored by Jacob Vosmaer's avatar Jacob Vosmaer

Use gitlab-zip-cat to send zip entries

parent adada74d
...@@ -2,3 +2,4 @@ gitlab-workhorse ...@@ -2,3 +2,4 @@ gitlab-workhorse
testdata/data testdata/data
testdata/scratch testdata/scratch
testdata/public testdata/public
gitlab-zip-cat
PREFIX=/usr/local PREFIX=/usr/local
VERSION=$(shell git describe)-$(shell date -u +%Y%m%d.%H%M%S) VERSION=$(shell git describe)-$(shell date -u +%Y%m%d.%H%M%S)
GOBUILD=go build -ldflags "-X main.Version=${VERSION}"
all: gitlab-zip-cat gitlab-workhorse
gitlab-zip-cat: $(shell find cmd/gitlab-zip-cat/ -name '*.go')
${GOBUILD} -o $@ ./cmd/$@
gitlab-workhorse: $(shell find . -name '*.go') gitlab-workhorse: $(shell find . -name '*.go')
go build -ldflags "-X main.Version=${VERSION}" -o gitlab-workhorse ${GOBUILD} -o $@
install: gitlab-workhorse install: gitlab-workhorse gitlab-zip-cat
install gitlab-workhorse ${PREFIX}/bin/ install gitlab-workhorse gitlab-zip-cat ${PREFIX}/bin/
.PHONY: test .PHONY: test
test: testdata/data/group/test.git clean-workhorse gitlab-workhorse test: testdata/data/group/test.git clean-workhorse gitlab-workhorse gitlab-zip-cat
go fmt ./... | awk '{ print } END { if (NR > 0) { print "Please run go fmt"; exit 1 } }' go fmt ./... | awk '{ print } END { if (NR > 0) { print "Please run go fmt"; exit 1 } }'
go test ./... support/path-add-current go test ./...
@echo SUCCESS @echo SUCCESS
coverage: testdata/data/group/test.git coverage: testdata/data/group/test.git
...@@ -30,4 +36,4 @@ clean: clean-workhorse ...@@ -30,4 +36,4 @@ clean: clean-workhorse
.PHONY: clean-workhorse .PHONY: clean-workhorse
clean-workhorse: clean-workhorse:
rm -f gitlab-workhorse rm -f gitlab-workhorse gitlab-zip-cat
package main
import (
"archive/zip"
"flag"
"fmt"
"io"
"log"
"os"
)
const notFound = 2
const progName = "gitlab-zip-cat"
var Version = "unknown"
var printVersion = flag.Bool("version", false, "Print version and exit")
func main() {
flag.Parse()
version := fmt.Sprintf("%s %s", progName, Version)
if *printVersion {
fmt.Println(version)
os.Exit(0)
}
archiveFileName := os.Args[1]
fileName := os.Args[2]
archive, err := zip.OpenReader(archiveFileName)
if err != nil {
printError(fmt.Errorf("open %q: %v", archiveFileName, err))
os.Exit(notFound)
}
defer archive.Close()
file := findFileInZip(fileName, &archive.Reader)
if file == nil {
printError(fmt.Errorf("find %q in %q: not found", fileName, archiveFileName))
os.Exit(notFound)
}
// Start decompressing the file
reader, err := file.Open()
if err != nil {
fatalError(fmt.Errorf("open %q in %q: %v", fileName, archiveFileName, err))
}
defer reader.Close()
if _, err := io.Copy(os.Stdout, reader); err != nil {
fatalError(fmt.Errorf("write %q from %q to stdout: %v", fileName, archiveFileName, err))
}
}
func findFileInZip(fileName string, archive *zip.Reader) *zip.File {
for _, file := range archive.File {
if file.Name == fileName {
return file
}
}
return nil
}
func printError(err error) {
log.Printf("%s: %v", progName, err)
}
func fatalError(err error) {
printError(err)
os.Exit(1)
}
...@@ -3,7 +3,6 @@ package artifacts ...@@ -3,7 +3,6 @@ package artifacts
import ( import (
"../api" "../api"
"../helper" "../helper"
"archive/zip"
"encoding/base64" "encoding/base64"
"errors" "errors"
"fmt" "fmt"
...@@ -11,10 +10,13 @@ import ( ...@@ -11,10 +10,13 @@ import (
"mime" "mime"
"net/http" "net/http"
"os" "os"
"os/exec"
"path/filepath" "path/filepath"
"strconv" "syscall"
) )
const exitStatusNotFound = 2
func decodeFileEntry(entry string) (string, error) { func decodeFileEntry(entry string) (string, error) {
decoded, err := base64.StdEncoding.DecodeString(entry) decoded, err := base64.StdEncoding.DecodeString(entry)
if err != nil { if err != nil {
...@@ -31,44 +33,38 @@ func detectFileContentType(fileName string) string { ...@@ -31,44 +33,38 @@ func detectFileContentType(fileName string) string {
return contentType return contentType
} }
func findFileInZip(fileName string, archive *zip.Reader) *zip.File {
for _, file := range archive.File {
if file.Name == fileName {
return file
}
}
return nil
}
func unpackFileFromZip(archiveFileName, fileName string, headers http.Header, output io.Writer) error { func unpackFileFromZip(archiveFileName, fileName string, headers http.Header, output io.Writer) error {
archive, err := zip.OpenReader(archiveFileName) catFile := exec.Command("gitlab-zip-cat", archiveFileName, fileName)
catFile.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
stdout, err := catFile.StdoutPipe()
if err != nil { if err != nil {
return err return fmt.Errorf("create gitlab-zip-cat stdout pipe: %v", err)
} }
defer archive.Close()
file := findFileInZip(fileName, &archive.Reader) if err := catFile.Start(); err != nil {
if file == nil { return fmt.Errorf("start %v: %v", catFile.Args, err)
return os.ErrNotExist
}
// Start decompressing the file
reader, err := file.Open()
if err != nil {
return err
} }
defer reader.Close() defer helper.CleanUpProcessGroup(catFile)
basename := filepath.Base(fileName) basename := filepath.Base(fileName)
// Write http headers about the file // Write http headers about the file
headers.Set("Content-Length", strconv.FormatInt(int64(file.UncompressedSize64), 10)) headers.Set("Content-Type", detectFileContentType(fileName))
headers.Set("Content-Type", detectFileContentType(file.Name))
headers.Set("Content-Disposition", "attachment; filename=\""+escapeQuotes(basename)+"\"") headers.Set("Content-Disposition", "attachment; filename=\""+escapeQuotes(basename)+"\"")
// Copy file body to client // Copy file body to client
_, err = io.Copy(output, reader) if _, err := io.Copy(output, stdout); err != nil {
return err return fmt.Errorf("copy %v stdout: %v", catFile.Args, err)
}
if err := catFile.Wait(); err != nil {
if st, ok := helper.ExitStatus(err); ok && st == exitStatusNotFound {
return os.ErrNotExist
}
return fmt.Errorf("wait for %v to finish: %v", catFile.Args, err)
}
return nil
} }
// Artifacts downloader doesn't support ranges when downloading a single file // Artifacts downloader doesn't support ranges when downloading a single file
......
...@@ -79,7 +79,7 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) { ...@@ -79,7 +79,7 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
helper.Fail500(w, fmt.Errorf("handleGetArchive: start %v: %v", archiveCmd.Args, err)) helper.Fail500(w, fmt.Errorf("handleGetArchive: start %v: %v", archiveCmd.Args, err))
return return
} }
defer cleanUpProcessGroup(archiveCmd) // Ensure brute force subprocess clean-up defer helper.CleanUpProcessGroup(archiveCmd) // Ensure brute force subprocess clean-up
var stdout io.ReadCloser var stdout io.ReadCloser
if compressCmd == nil { if compressCmd == nil {
...@@ -99,7 +99,7 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) { ...@@ -99,7 +99,7 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
helper.Fail500(w, fmt.Errorf("handleGetArchive: start %v: %v", compressCmd.Args, err)) helper.Fail500(w, fmt.Errorf("handleGetArchive: start %v: %v", compressCmd.Args, err))
return return
} }
defer cleanUpProcessGroup(compressCmd) defer helper.CleanUpProcessGroup(compressCmd)
archiveStdout.Close() archiveStdout.Close()
} }
......
...@@ -23,18 +23,3 @@ func gitCommand(gl_id string, name string, args ...string) *exec.Cmd { ...@@ -23,18 +23,3 @@ func gitCommand(gl_id string, name string, args ...string) *exec.Cmd {
cmd.Stderr = os.Stderr cmd.Stderr = os.Stderr
return cmd return cmd
} }
func cleanUpProcessGroup(cmd *exec.Cmd) {
if cmd == nil {
return
}
process := cmd.Process
if process != nil && process.Pid > 0 {
// Send SIGTERM to the process group of cmd
syscall.Kill(-process.Pid, syscall.SIGTERM)
}
// reap our child process
cmd.Wait()
}
...@@ -72,7 +72,7 @@ func handleGetInfoRefs(w http.ResponseWriter, r *http.Request, a *api.Response) ...@@ -72,7 +72,7 @@ func handleGetInfoRefs(w http.ResponseWriter, r *http.Request, a *api.Response)
helper.Fail500(w, fmt.Errorf("handleGetInfoRefs: start %v: %v", cmd.Args, err)) helper.Fail500(w, fmt.Errorf("handleGetInfoRefs: start %v: %v", cmd.Args, err))
return return
} }
defer cleanUpProcessGroup(cmd) // Ensure brute force subprocess clean-up defer helper.CleanUpProcessGroup(cmd) // Ensure brute force subprocess clean-up
// Start writing the response // Start writing the response
w.Header().Add("Content-Type", fmt.Sprintf("application/x-%s-advertisement", rpc)) w.Header().Add("Content-Type", fmt.Sprintf("application/x-%s-advertisement", rpc))
...@@ -125,7 +125,7 @@ func handlePostRPC(w http.ResponseWriter, r *http.Request, a *api.Response) { ...@@ -125,7 +125,7 @@ func handlePostRPC(w http.ResponseWriter, r *http.Request, a *api.Response) {
helper.Fail500(w, fmt.Errorf("handlePostRPC: start %v: %v", cmd.Args, err)) helper.Fail500(w, fmt.Errorf("handlePostRPC: start %v: %v", cmd.Args, err))
return return
} }
defer cleanUpProcessGroup(cmd) // Ensure brute force subprocess clean-up defer helper.CleanUpProcessGroup(cmd) // Ensure brute force subprocess clean-up
// Write the client request body to Git's standard input // Write the client request body to Git's standard input
if _, err := io.Copy(stdin, r.Body); err != nil { if _, err := io.Copy(stdin, r.Body); err != nil {
......
...@@ -6,6 +6,8 @@ import ( ...@@ -6,6 +6,8 @@ import (
"net/http" "net/http"
"net/url" "net/url"
"os" "os"
"os/exec"
"syscall"
) )
func Fail500(w http.ResponseWriter, err error) { func Fail500(w http.ResponseWriter, err error) {
...@@ -69,3 +71,32 @@ func HTTPError(w http.ResponseWriter, r *http.Request, error string, code int) { ...@@ -69,3 +71,32 @@ func HTTPError(w http.ResponseWriter, r *http.Request, error string, code int) {
http.Error(w, error, code) http.Error(w, error, code)
} }
func CleanUpProcessGroup(cmd *exec.Cmd) {
if cmd == nil {
return
}
process := cmd.Process
if process != nil && process.Pid > 0 {
// Send SIGTERM to the process group of cmd
syscall.Kill(-process.Pid, syscall.SIGTERM)
}
// reap our child process
cmd.Wait()
}
func ExitStatus(err error) (int, bool) {
exitError, ok := err.(*exec.ExitError)
if !ok {
return 0, false
}
waitStatus, ok := exitError.Sys().(syscall.WaitStatus)
if !ok {
return 0, false
}
return waitStatus.ExitStatus(), true
}
#!/bin/sh
exec env PATH=$(pwd):${PATH} "$@"
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment