Commit adf7680a authored by Jacob Vosmaer's avatar Jacob Vosmaer

Merge branch 'master' of https://gitlab.com/gitlab-org/gitlab-workhorse into archive-header

parents 1a9126c9 3f8da4ae
......@@ -2,6 +2,20 @@
Formerly known as 'gitlab-git-http-server'.
0.6.4
Increase default ProxyHeadersTimeout to 5 minutes. Fix injecting raw
blobs for /api/v3 requetsts.
0.6.3
Add support for sending Git raw git blobs via gitlab-workhorse.
0.6.2
We now fill in missing directory entries in archize zip metadata
files; also some other minor changes.
0.6.1
Add support for generating zip artifacts metadata and serving single
......
......@@ -33,7 +33,7 @@ Options:
Print version and exit
```
The 'auth backend' refers to the GitLab Rails applicatoin. The name is
The 'auth backend' refers to the GitLab Rails application. The name is
a holdover from when gitlab-workhorse only handled Git push/pull over
HTTP.
......
......@@ -23,7 +23,7 @@ func main() {
}
if len(os.Args) != 2 {
fmt.Fprintf(os.Stderr, "Usage: %s FILE.ZIP", progName)
fmt.Fprintf(os.Stderr, "Usage: %s FILE.ZIP\n", progName)
os.Exit(1)
}
if err := zipartifacts.GenerateZipMetadataFromFile(os.Args[1], os.Stdout); err != nil {
......
......@@ -35,6 +35,8 @@ func (b *blob) Inject(w http.ResponseWriter, r *http.Request, sendData string) {
}
defer helper.CleanUpProcessGroup(gitShowCmd)
// Ignore incorrect Content-Length that may have been set by Rails
w.Header().Del("Content-Length")
if _, err := io.Copy(w, stdout); err != nil {
helper.LogError(fmt.Errorf("SendBlob: copy git cat-file stdout: %v", err))
return
......
......@@ -7,12 +7,14 @@ import (
"encoding/json"
"io"
"os"
"path"
"sort"
"strconv"
)
type metadata struct {
Modified int64 `json:"modified"`
Mode string `json:"mode"`
Modified int64 `json:"modified,omitempty"`
Mode string `json:"mode,omitempty"`
CRC uint32 `json:"crc,omitempty"`
Size uint64 `json:"size,omitempty"`
Zipped uint64 `json:"zipped,omitempty"`
......@@ -23,6 +25,10 @@ const MetadataHeaderPrefix = "\x00\x00\x00&" // length of string below, encoded
const MetadataHeader = "GitLab Build Artifacts Metadata 0.0.2\n"
func newMetadata(file *zip.File) metadata {
if file == nil {
return metadata{}
}
return metadata{
Modified: file.ModTime().Unix(),
Mode: strconv.FormatUint(uint64(file.Mode().Perm()), 8),
......@@ -42,35 +48,53 @@ func (m metadata) writeEncoded(output io.Writer) error {
return writeBytes(output, j)
}
func writeZipEntryMetadata(output io.Writer, entry *zip.File) error {
err := writeString(output, entry.Name)
if err != nil {
func writeZipEntryMetadata(output io.Writer, path string, entry *zip.File) error {
if err := writeString(output, path); err != nil {
return err
}
err = newMetadata(entry).writeEncoded(output)
if err != nil {
if err := newMetadata(entry).writeEncoded(output); err != nil {
return err
}
return nil
}
func generateZipMetadata(output io.Writer, archive *zip.Reader) error {
err := writeString(output, MetadataHeader)
if err != nil {
if err := writeString(output, MetadataHeader); err != nil {
return err
}
// Write empty error string
err = writeString(output, "{}")
if err != nil {
// Write empty error header that we may need in the future
if err := writeString(output, "{}"); err != nil {
return err
}
// Write all files
// Create map of files in zip archive
zipMap := make(map[string]*zip.File, len(archive.File))
// Add missing entries
for _, entry := range archive.File {
err = writeZipEntryMetadata(output, entry)
if err != nil {
zipMap[entry.Name] = entry
for d := path.Dir(entry.Name); d != "." && d != "/"; d = path.Dir(d) {
entryDir := d + "/"
if _, ok := zipMap[entryDir]; !ok {
zipMap[entryDir] = nil
}
}
}
// Sort paths
sortedPaths := make([]string, 0, len(zipMap))
for path, _ := range zipMap {
sortedPaths = append(sortedPaths, path)
}
sort.Strings(sortedPaths)
// Write all files
for _, path := range sortedPaths {
if err := writeZipEntryMetadata(output, path, zipMap[path]); err != nil {
return err
}
}
......
package zipartifacts
import (
"archive/zip"
"bytes"
"encoding/binary"
"fmt"
"testing"
)
func TestMissingMetadataEntries(t *testing.T) {
var zipBuffer, metaBuffer bytes.Buffer
archive := zip.NewWriter(&zipBuffer)
// non-POSIX paths are here just to test if we never enter infinite loop
files := []string{"file1", "some/file/dir/", "some/file/dir/file2", "../../test12/test",
"/usr/bin/test", `c:\windows\win32.exe`, `c:/windows/win.dll`, "./f/asd", "/"}
for _, file := range files {
archiveFile, err := archive.Create(file)
if err != nil {
t.Fatal(err)
}
fmt.Fprint(archiveFile, file)
}
archive.Close()
zipReader := bytes.NewReader(zipBuffer.Bytes())
zipArchiveReader, _ := zip.NewReader(zipReader, int64(binary.Size(zipBuffer.Bytes())))
if err := generateZipMetadata(&metaBuffer, zipArchiveReader); err != nil {
t.Fatal("zipartifacts: generateZipMetadata failed", err)
}
paths := []string{"file1", "some/", "some/file/", "some/file/dir/", "some/file/dir/file2"}
for _, path := range paths {
if !bytes.Contains(metaBuffer.Bytes(), []byte(path+"\x00")) {
t.Fatal("zipartifacts: metadata for path", path, "not found")
}
}
}
......@@ -37,7 +37,7 @@ var authBackend = URLFlag("authBackend", upstream.DefaultBackend, "Authenticatio
var authSocket = flag.String("authSocket", "", "Optional: Unix domain socket to dial authBackend at")
var pprofListenAddr = flag.String("pprofListenAddr", "", "pprof listening address, e.g. 'localhost:6060'")
var documentRoot = flag.String("documentRoot", "public", "Path to static files content")
var proxyHeadersTimeout = flag.Duration("proxyHeadersTimeout", time.Minute, "How long to wait for response headers when proxying the request")
var proxyHeadersTimeout = flag.Duration("proxyHeadersTimeout", 5*time.Minute, "How long to wait for response headers when proxying the request")
var developmentMode = flag.Bool("developmentMode", false, "Allow to serve assets from Rails app")
func main() {
......
......@@ -563,8 +563,6 @@ func TestGetGitBlob(t *testing.T) {
responseJSON := fmt.Sprintf(`{"RepoPath":"%s","BlobId":"%s"}`, path.Join(testRepoRoot, testRepo), blobId)
encodedJSON := base64.StdEncoding.EncodeToString([]byte(responseJSON))
w.Header().Set(headerKey, "git-blob:"+encodedJSON)
// Prevent the Go HTTP server from setting the Content-Length to 0.
w.Header().Set("Transfer-Encoding", "chunked")
if _, err := fmt.Fprintf(w, "GNU General Public License"); err != nil {
t.Fatal(err)
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment