Commit 8aa52cff authored by Jacob Vosmaer's avatar Jacob Vosmaer

Generate zip metadata in a subprocess

parent 1629812b
......@@ -2,11 +2,14 @@ PREFIX=/usr/local
VERSION=$(shell git describe)-$(shell date -u +%Y%m%d.%H%M%S)
GOBUILD=go build -ldflags "-X main.Version=${VERSION}"
all: gitlab-zip-cat gitlab-workhorse
all: gitlab-zip-cat gitlab-zip-metadata gitlab-workhorse
gitlab-zip-cat: $(shell find cmd/gitlab-zip-cat/ -name '*.go')
${GOBUILD} -o $@ ./cmd/$@
gitlab-zip-metadata: $(shell find cmd/gitlab-zip-metadata/ -name '*.go')
${GOBUILD} -o $@ ./cmd/$@
gitlab-workhorse: $(shell find . -name '*.go')
${GOBUILD} -o $@
......@@ -14,7 +17,7 @@ install: gitlab-workhorse gitlab-zip-cat
install gitlab-workhorse gitlab-zip-cat ${PREFIX}/bin/
.PHONY: test
test: testdata/data/group/test.git clean-workhorse gitlab-workhorse gitlab-zip-cat
test: testdata/data/group/test.git clean-workhorse all
go fmt ./... | awk '{ print } END { if (NR > 0) { print "Please run go fmt"; exit 1 } }'
support/path-add-current go test ./...
@echo SUCCESS
......@@ -36,4 +39,4 @@ clean: clean-workhorse
.PHONY: clean-workhorse
clean-workhorse:
rm -f gitlab-workhorse gitlab-zip-cat
rm -f gitlab-workhorse gitlab-zip-cat gitlab-zip-metadata
......@@ -25,6 +25,11 @@ func main() {
os.Exit(0)
}
if len(os.Args) != 3 {
fmt.Fprintf(os.Stderr, "Usage: %s FILE.ZIP ENTRY", progName)
os.Exit(1)
}
archiveFileName := os.Args[1]
fileName := os.Args[2]
archive, err := zip.OpenReader(archiveFileName)
......
package main
import (
"../../internal/zipmetadata"
"flag"
"fmt"
"os"
)
const progName = "gitlab-zip-metadata"
var Version = "unknown"
var printVersion = flag.Bool("version", false, "Print version and exit")
func main() {
flag.Parse()
version := fmt.Sprintf("%s %s", progName, Version)
if *printVersion {
fmt.Println(version)
os.Exit(0)
}
if len(os.Args) != 2 {
fmt.Fprintf(os.Stderr, "Usage: %s FILE.ZIP", progName)
os.Exit(1)
}
if err := zipmetadata.GenerateZipMetadataFromFile(os.Args[1], os.Stdout); err != nil {
fmt.Fprintf(os.Stderr, "%s: %v\n", progName, err)
if err == os.ErrInvalid {
os.Exit(zipmetadata.StatusNotZip)
}
os.Exit(1)
}
}
......@@ -39,6 +39,7 @@ func detectFileContentType(fileName string) string {
func unpackFileFromZip(archiveFileName, fileName string, headers http.Header, output io.Writer) error {
catFile := exec.Command("gitlab-zip-cat", archiveFileName, fileName)
catFile.Stderr = os.Stderr
catFile.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
stdout, err := catFile.StdoutPipe()
if err != nil {
......
......@@ -4,12 +4,15 @@ import (
"../api"
"../helper"
"../upload"
"../zipmetadata"
"errors"
"fmt"
"io/ioutil"
"mime/multipart"
"net/http"
"os"
"os/exec"
"syscall"
)
type artifactsUploadProcessor struct {
......@@ -36,10 +39,19 @@ func (a *artifactsUploadProcessor) ProcessFile(formName, fileName string, writer
a.metadataFile = tempFile.Name()
// Generate metadata and save to file
err = generateZipMetadataFromFile(fileName, tempFile)
if err == os.ErrInvalid {
return nil
} else if err != nil {
zipMd := exec.Command("gitlab-zip-metadata", fileName)
zipMd.Stderr = os.Stderr
zipMd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
zipMd.Stdout = tempFile
if err := zipMd.Start(); err != nil {
return err
}
defer helper.CleanUpProcessGroup(zipMd)
if err := zipMd.Wait(); err != nil {
if st, ok := helper.ExitStatus(err); ok && st == zipmetadata.StatusNotZip {
return nil
}
return err
}
......
......@@ -5,6 +5,7 @@ import (
"../helper"
"../proxy"
"../testhelper"
"../zipmetadata"
"archive/zip"
"bytes"
"compress/gzip"
......@@ -71,7 +72,7 @@ func testArtifactsUploadServer(t *testing.T, tempPath string) *httptest.Server {
w.WriteHeader(404)
return
}
if !bytes.HasPrefix(metadata, []byte(metadataHeaderPrefix+metadataHeader)) {
if !bytes.HasPrefix(metadata, []byte(zipmetadata.HeaderPrefix+zipmetadata.Header)) {
w.WriteHeader(400)
return
}
......
package artifacts
package zipmetadata
import (
"archive/zip"
......@@ -19,8 +19,9 @@ type metadata struct {
Comment string `json:"comment,omitempty"`
}
const metadataHeaderPrefix = "\x00\x00\x00&" // length of string below, encoded properly
const metadataHeader = "GitLab Build Artifacts Metadata 0.0.2\n"
const HeaderPrefix = "\x00\x00\x00&" // length of string below, encoded properly
const Header = "GitLab Build Artifacts Metadata 0.0.2\n"
const StatusNotZip = 2
func newMetadata(file *zip.File) metadata {
return metadata{
......@@ -56,7 +57,7 @@ func writeZipEntryMetadata(output io.Writer, entry *zip.File) error {
}
func generateZipMetadata(output io.Writer, archive *zip.Reader) error {
err := writeString(output, metadataHeader)
err := writeString(output, Header)
if err != nil {
return err
}
......@@ -77,7 +78,7 @@ func generateZipMetadata(output io.Writer, archive *zip.Reader) error {
return nil
}
func generateZipMetadataFromFile(fileName string, w io.Writer) error {
func GenerateZipMetadataFromFile(fileName string, w io.Writer) error {
archive, err := zip.OpenReader(fileName)
if err != nil {
// Ignore non-zip archives
......
......@@ -466,8 +466,9 @@ func TestArtifactsUpload(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if len(r.MultipartForm.Value) != 2 { // 1 file name, 1 file path
t.Error("Expected to receive exactly 2 values")
nValues := 2 // filename + path for just the upload (no metadata because we are not POSTing a valid zip file)
if len(r.MultipartForm.Value) != nValues {
t.Errorf("Expected to receive exactly %d values", nValues)
}
if len(r.MultipartForm.File) != 0 {
t.Error("Expected to not receive any files")
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment