Merge branch 'master' of https://gitlab.com/gitlab-org/gitlab-workhorse into archive-header

CHANGELOG

@@ -2,6 +2,20 @@
 
 Formerly known as 'gitlab-git-http-server'.
 
+0.6.4
+
+Increase default ProxyHeadersTimeout to 5 minutes. Fix injecting raw
+blobs for /api/v3 requetsts.
+
+0.6.3
+
+Add support for sending Git raw git blobs via gitlab-workhorse.
+
+0.6.2
+
+We now fill in missing directory entries in archize zip metadata
+files; also some other minor changes.
+
 0.6.1
 
 Add support for generating zip artifacts metadata and serving single

README.md

@@ -33,7 +33,7 @@ Options:
     	Print version and exit
 ```
 
-The 'auth backend' refers to the GitLab Rails applicatoin. The name is
+The 'auth backend' refers to the GitLab Rails application. The name is
 a holdover from when gitlab-workhorse only handled Git push/pull over
 HTTP.
 

VERSION

-0.6.1
+0.6.4

cmd/gitlab-zip-metadata/main.go

@@ -23,7 +23,7 @@ func main() {
 	}
 
 	if len(os.Args) != 2 {
-		fmt.Fprintf(os.Stderr, "Usage: %s FILE.ZIP", progName)
+		fmt.Fprintf(os.Stderr, "Usage: %s FILE.ZIP\n", progName)
 		os.Exit(1)
 	}
 	if err := zipartifacts.GenerateZipMetadataFromFile(os.Args[1], os.Stdout); err != nil {

internal/git/blob.go

@@ -35,6 +35,8 @@ func (b *blob) Inject(w http.ResponseWriter, r *http.Request, sendData string) {
 	}
 	defer helper.CleanUpProcessGroup(gitShowCmd)
 
+	// Ignore incorrect Content-Length that may have been set by Rails
+	w.Header().Del("Content-Length")
 	if _, err := io.Copy(w, stdout); err != nil {
 		helper.LogError(fmt.Errorf("SendBlob: copy git cat-file stdout: %v", err))
 		return

internal/zipartifacts/metadata.go

@@ -7,12 +7,14 @@ import (
 	"encoding/json"
 	"io"
 	"os"
+	"path"
+	"sort"
 	"strconv"
 )
 
 type metadata struct {
-	Modified int64  `json:"modified"`
-	Mode     string `json:"mode"`
+	Modified int64  `json:"modified,omitempty"`
+	Mode     string `json:"mode,omitempty"`
 	CRC      uint32 `json:"crc,omitempty"`
 	Size     uint64 `json:"size,omitempty"`
 	Zipped   uint64 `json:"zipped,omitempty"`
@@ -23,6 +25,10 @@ const MetadataHeaderPrefix = "\x00\x00\x00&" // length of string below, encoded
 const MetadataHeader = "GitLab Build Artifacts Metadata 0.0.2\n"
 
 func newMetadata(file *zip.File) metadata {
+	if file == nil {
+		return metadata{}
+	}
+
 	return metadata{
 		Modified: file.ModTime().Unix(),
 		Mode:     strconv.FormatUint(uint64(file.Mode().Perm()), 8),
@@ -42,35 +48,53 @@ func (m metadata) writeEncoded(output io.Writer) error {
 	return writeBytes(output, j)
 }
 
-func writeZipEntryMetadata(output io.Writer, entry *zip.File) error {
-	err := writeString(output, entry.Name)
-	if err != nil {
+func writeZipEntryMetadata(output io.Writer, path string, entry *zip.File) error {
+	if err := writeString(output, path); err != nil {
 		return err
 	}
 
-	err = newMetadata(entry).writeEncoded(output)
-	if err != nil {
+	if err := newMetadata(entry).writeEncoded(output); err != nil {
 		return err
 	}
+
 	return nil
 }
 
 func generateZipMetadata(output io.Writer, archive *zip.Reader) error {
-	err := writeString(output, MetadataHeader)
-	if err != nil {
+	if err := writeString(output, MetadataHeader); err != nil {
 		return err
 	}
 
-	// Write empty error string
-	err = writeString(output, "{}")
-	if err != nil {
+	// Write empty error header that we may need in the future
+	if err := writeString(output, "{}"); err != nil {
 		return err
 	}
 
-	// Write all files
+	// Create map of files in zip archive
+	zipMap := make(map[string]*zip.File, len(archive.File))
+
+	// Add missing entries
 	for _, entry := range archive.File {
-		err = writeZipEntryMetadata(output, entry)
-		if err != nil {
+		zipMap[entry.Name] = entry
+
+		for d := path.Dir(entry.Name); d != "." && d != "/"; d = path.Dir(d) {
+			entryDir := d + "/"
+			if _, ok := zipMap[entryDir]; !ok {
+				zipMap[entryDir] = nil
+			}
+		}
+	}
+
+	// Sort paths
+	sortedPaths := make([]string, 0, len(zipMap))
+	for path, _ := range zipMap {
+		sortedPaths = append(sortedPaths, path)
+	}
+	sort.Strings(sortedPaths)
+
+	// Write all files
+	for _, path := range sortedPaths {
+		if err := writeZipEntryMetadata(output, path, zipMap[path]); err != nil {
 			return err
 		}
 	}

internal/zipartifacts/metadata_test.go

+package zipartifacts
+
+import (
+	"archive/zip"
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"testing"
+)
+
+func TestMissingMetadataEntries(t *testing.T) {
+	var zipBuffer, metaBuffer bytes.Buffer
+
+	archive := zip.NewWriter(&zipBuffer)
+
+	// non-POSIX paths are here just to test if we never enter infinite loop
+	files := []string{"file1", "some/file/dir/", "some/file/dir/file2", "../../test12/test",
+		"/usr/bin/test", `c:\windows\win32.exe`, `c:/windows/win.dll`, "./f/asd", "/"}
+
+	for _, file := range files {
+		archiveFile, err := archive.Create(file)
+		if err != nil {
+			t.Fatal(err)
+		}
+		fmt.Fprint(archiveFile, file)
+	}
+
+	archive.Close()
+
+	zipReader := bytes.NewReader(zipBuffer.Bytes())
+	zipArchiveReader, _ := zip.NewReader(zipReader, int64(binary.Size(zipBuffer.Bytes())))
+	if err := generateZipMetadata(&metaBuffer, zipArchiveReader); err != nil {
+		t.Fatal("zipartifacts: generateZipMetadata failed", err)
+	}
+
+	paths := []string{"file1", "some/", "some/file/", "some/file/dir/", "some/file/dir/file2"}
+	for _, path := range paths {
+		if !bytes.Contains(metaBuffer.Bytes(), []byte(path+"\x00")) {
+			t.Fatal("zipartifacts: metadata for path", path, "not found")
+		}
+	}
+}

main.go

@@ -37,7 +37,7 @@ var authBackend = URLFlag("authBackend", upstream.DefaultBackend, "Authenticatio
 var authSocket = flag.String("authSocket", "", "Optional: Unix domain socket to dial authBackend at")
 var pprofListenAddr = flag.String("pprofListenAddr", "", "pprof listening address, e.g. 'localhost:6060'")
 var documentRoot = flag.String("documentRoot", "public", "Path to static files content")
-var proxyHeadersTimeout = flag.Duration("proxyHeadersTimeout", time.Minute, "How long to wait for response headers when proxying the request")
+var proxyHeadersTimeout = flag.Duration("proxyHeadersTimeout", 5*time.Minute, "How long to wait for response headers when proxying the request")
 var developmentMode = flag.Bool("developmentMode", false, "Allow to serve assets from Rails app")
 
 func main() {

main_test.go

@@ -563,8 +563,6 @@ func TestGetGitBlob(t *testing.T) {
 		responseJSON := fmt.Sprintf(`{"RepoPath":"%s","BlobId":"%s"}`, path.Join(testRepoRoot, testRepo), blobId)
 		encodedJSON := base64.StdEncoding.EncodeToString([]byte(responseJSON))
 		w.Header().Set(headerKey, "git-blob:"+encodedJSON)
-		// Prevent the Go HTTP server from setting the Content-Length to 0.
-		w.Header().Set("Transfer-Encoding", "chunked")
 		if _, err := fmt.Fprintf(w, "GNU General Public License"); err != nil {
 			t.Fatal(err)
 		}