Commit 1ccba44f authored by Jacob Vosmaer's avatar Jacob Vosmaer

Use gitlab-zip-cat to send zip entries

parent adada74d
......@@ -2,3 +2,4 @@ gitlab-workhorse
testdata/data
testdata/scratch
testdata/public
gitlab-zip-cat
PREFIX=/usr/local
VERSION=$(shell git describe)-$(shell date -u +%Y%m%d.%H%M%S)
GOBUILD=go build -ldflags "-X main.Version=${VERSION}"
all: gitlab-zip-cat gitlab-workhorse
gitlab-zip-cat: $(shell find cmd/gitlab-zip-cat/ -name '*.go')
${GOBUILD} -o $@ ./cmd/$@
gitlab-workhorse: $(shell find . -name '*.go')
go build -ldflags "-X main.Version=${VERSION}" -o gitlab-workhorse
${GOBUILD} -o $@
install: gitlab-workhorse
install gitlab-workhorse ${PREFIX}/bin/
install: gitlab-workhorse gitlab-zip-cat
install gitlab-workhorse gitlab-zip-cat ${PREFIX}/bin/
.PHONY: test
test: testdata/data/group/test.git clean-workhorse gitlab-workhorse
test: testdata/data/group/test.git clean-workhorse gitlab-workhorse gitlab-zip-cat
go fmt ./... | awk '{ print } END { if (NR > 0) { print "Please run go fmt"; exit 1 } }'
go test ./...
support/path-add-current go test ./...
@echo SUCCESS
coverage: testdata/data/group/test.git
......@@ -30,4 +36,4 @@ clean: clean-workhorse
.PHONY: clean-workhorse
clean-workhorse:
rm -f gitlab-workhorse
rm -f gitlab-workhorse gitlab-zip-cat
package main
import (
"archive/zip"
"flag"
"fmt"
"io"
"log"
"os"
)
const notFound = 2
const progName = "gitlab-zip-cat"
var Version = "unknown"
var printVersion = flag.Bool("version", false, "Print version and exit")
func main() {
flag.Parse()
version := fmt.Sprintf("%s %s", progName, Version)
if *printVersion {
fmt.Println(version)
os.Exit(0)
}
archiveFileName := os.Args[1]
fileName := os.Args[2]
archive, err := zip.OpenReader(archiveFileName)
if err != nil {
printError(fmt.Errorf("open %q: %v", archiveFileName, err))
os.Exit(notFound)
}
defer archive.Close()
file := findFileInZip(fileName, &archive.Reader)
if file == nil {
printError(fmt.Errorf("find %q in %q: not found", fileName, archiveFileName))
os.Exit(notFound)
}
// Start decompressing the file
reader, err := file.Open()
if err != nil {
fatalError(fmt.Errorf("open %q in %q: %v", fileName, archiveFileName, err))
}
defer reader.Close()
if _, err := io.Copy(os.Stdout, reader); err != nil {
fatalError(fmt.Errorf("write %q from %q to stdout: %v", fileName, archiveFileName, err))
}
}
func findFileInZip(fileName string, archive *zip.Reader) *zip.File {
for _, file := range archive.File {
if file.Name == fileName {
return file
}
}
return nil
}
func printError(err error) {
log.Printf("%s: %v", progName, err)
}
func fatalError(err error) {
printError(err)
os.Exit(1)
}
......@@ -3,7 +3,6 @@ package artifacts
import (
"../api"
"../helper"
"archive/zip"
"encoding/base64"
"errors"
"fmt"
......@@ -11,10 +10,13 @@ import (
"mime"
"net/http"
"os"
"os/exec"
"path/filepath"
"strconv"
"syscall"
)
const exitStatusNotFound = 2
func decodeFileEntry(entry string) (string, error) {
decoded, err := base64.StdEncoding.DecodeString(entry)
if err != nil {
......@@ -31,44 +33,38 @@ func detectFileContentType(fileName string) string {
return contentType
}
func findFileInZip(fileName string, archive *zip.Reader) *zip.File {
for _, file := range archive.File {
if file.Name == fileName {
return file
}
}
return nil
}
func unpackFileFromZip(archiveFileName, fileName string, headers http.Header, output io.Writer) error {
archive, err := zip.OpenReader(archiveFileName)
catFile := exec.Command("gitlab-zip-cat", archiveFileName, fileName)
catFile.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
stdout, err := catFile.StdoutPipe()
if err != nil {
return err
return fmt.Errorf("create gitlab-zip-cat stdout pipe: %v", err)
}
defer archive.Close()
file := findFileInZip(fileName, &archive.Reader)
if file == nil {
return os.ErrNotExist
}
// Start decompressing the file
reader, err := file.Open()
if err != nil {
return err
if err := catFile.Start(); err != nil {
return fmt.Errorf("start %v: %v", catFile.Args, err)
}
defer reader.Close()
defer helper.CleanUpProcessGroup(catFile)
basename := filepath.Base(fileName)
// Write http headers about the file
headers.Set("Content-Length", strconv.FormatInt(int64(file.UncompressedSize64), 10))
headers.Set("Content-Type", detectFileContentType(file.Name))
headers.Set("Content-Type", detectFileContentType(fileName))
headers.Set("Content-Disposition", "attachment; filename=\""+escapeQuotes(basename)+"\"")
// Copy file body to client
_, err = io.Copy(output, reader)
return err
if _, err := io.Copy(output, stdout); err != nil {
return fmt.Errorf("copy %v stdout: %v", catFile.Args, err)
}
if err := catFile.Wait(); err != nil {
if st, ok := helper.ExitStatus(err); ok && st == exitStatusNotFound {
return os.ErrNotExist
}
return fmt.Errorf("wait for %v to finish: %v", catFile.Args, err)
}
return nil
}
// Artifacts downloader doesn't support ranges when downloading a single file
......
......@@ -79,7 +79,7 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
helper.Fail500(w, fmt.Errorf("handleGetArchive: start %v: %v", archiveCmd.Args, err))
return
}
defer cleanUpProcessGroup(archiveCmd) // Ensure brute force subprocess clean-up
defer helper.CleanUpProcessGroup(archiveCmd) // Ensure brute force subprocess clean-up
var stdout io.ReadCloser
if compressCmd == nil {
......@@ -99,7 +99,7 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
helper.Fail500(w, fmt.Errorf("handleGetArchive: start %v: %v", compressCmd.Args, err))
return
}
defer cleanUpProcessGroup(compressCmd)
defer helper.CleanUpProcessGroup(compressCmd)
archiveStdout.Close()
}
......
......@@ -23,18 +23,3 @@ func gitCommand(gl_id string, name string, args ...string) *exec.Cmd {
cmd.Stderr = os.Stderr
return cmd
}
func cleanUpProcessGroup(cmd *exec.Cmd) {
if cmd == nil {
return
}
process := cmd.Process
if process != nil && process.Pid > 0 {
// Send SIGTERM to the process group of cmd
syscall.Kill(-process.Pid, syscall.SIGTERM)
}
// reap our child process
cmd.Wait()
}
......@@ -72,7 +72,7 @@ func handleGetInfoRefs(w http.ResponseWriter, r *http.Request, a *api.Response)
helper.Fail500(w, fmt.Errorf("handleGetInfoRefs: start %v: %v", cmd.Args, err))
return
}
defer cleanUpProcessGroup(cmd) // Ensure brute force subprocess clean-up
defer helper.CleanUpProcessGroup(cmd) // Ensure brute force subprocess clean-up
// Start writing the response
w.Header().Add("Content-Type", fmt.Sprintf("application/x-%s-advertisement", rpc))
......@@ -125,7 +125,7 @@ func handlePostRPC(w http.ResponseWriter, r *http.Request, a *api.Response) {
helper.Fail500(w, fmt.Errorf("handlePostRPC: start %v: %v", cmd.Args, err))
return
}
defer cleanUpProcessGroup(cmd) // Ensure brute force subprocess clean-up
defer helper.CleanUpProcessGroup(cmd) // Ensure brute force subprocess clean-up
// Write the client request body to Git's standard input
if _, err := io.Copy(stdin, r.Body); err != nil {
......
......@@ -6,6 +6,8 @@ import (
"net/http"
"net/url"
"os"
"os/exec"
"syscall"
)
func Fail500(w http.ResponseWriter, err error) {
......@@ -69,3 +71,32 @@ func HTTPError(w http.ResponseWriter, r *http.Request, error string, code int) {
http.Error(w, error, code)
}
func CleanUpProcessGroup(cmd *exec.Cmd) {
if cmd == nil {
return
}
process := cmd.Process
if process != nil && process.Pid > 0 {
// Send SIGTERM to the process group of cmd
syscall.Kill(-process.Pid, syscall.SIGTERM)
}
// reap our child process
cmd.Wait()
}
func ExitStatus(err error) (int, bool) {
exitError, ok := err.(*exec.ExitError)
if !ok {
return 0, false
}
waitStatus, ok := exitError.Sys().(syscall.WaitStatus)
if !ok {
return 0, false
}
return waitStatus.ExitStatus(), true
}
#!/bin/sh
exec env PATH=$(pwd):${PATH} "$@"
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment