Introduce naive (using the same process) artifacts metadata generator and downloader

internal/api/api.go

@@ -58,6 +58,10 @@ type Response struct {
 	// TmpPath is the path where we should store temporary files
 	// This is set by authorization middleware
 	TempPath string
+	// Archive is the path where the artifacts archive is stored
+	Archive string `json:"archive"`
+	// Path is the filename inside the archive to extracted file
+	Path string `json:"path"`
 }
 
 // singleJoiningSlash is taken from reverseproxy.go:NewSingleHostReverseProxy

internal/artifacts/artifacts.go

+package artifacts
+
+import (
+	"../api"
+	"../upload"
+	"net/http"
+	"../helper"
+	"errors"
+	"os"
+	"archive/zip"
+	"encoding/base64"
+	"strconv"
+	"mime"
+	"path/filepath"
+	"io"
+)
+
+func UploadArtifacts(myAPI *api.API, h http.Handler) http.Handler {
+	return myAPI.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) {
+		if a.TempPath == "" {
+			helper.Fail500(w, errors.New("UploadArtifacts: TempPath is empty"))
+			return
+		}
+
+		upload.HandleFileUploads(w, r, h, a.TempPath, &artifactsFormFilter{})
+	}, "/authorize")
+}
+
+// Artifacts downloader doesn't support ranges when downloading a single file
+func DownloadArtifact(myAPI *api.API) http.Handler {
+	return myAPI.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) {
+		if a.Archive == "" || a.Path == "" {
+			helper.Fail500(w, errors.New("DownloadArtifact: Archive or Path is empty"))
+			return
+		}
+
+		fileNameDecoded, err := base64.StdEncoding.DecodeString(a.Path)
+		if err != nil {
+			helper.Fail500(w, err)
+			return
+		}
+		fileName := string(fileNameDecoded)
+
+		// TODO:
+		// This should be moved to sub process to reduce memory pressue on workhorse
+		archive, err := zip.OpenReader(a.Archive)
+		if os.IsNotExist(err) {
+			http.NotFound(w, r)
+			return
+		} else if err != nil {
+			helper.Fail500(w, err)
+		}
+		defer archive.Close()
+
+		var file *zip.File
+		for _, file = range archive.File {
+			if file.Name == fileName {
+				break
+			}
+		}
+		if file == nil {
+			http.NotFound(w, r)
+			return
+		}
+
+		contentType := mime.TypeByExtension(filepath.Ext(file.Name))
+		if contentType == "" {
+			contentType = "application/octet-stream"
+		}
+
+		w.Header().Set("Content-Length", strconv.FormatInt(int64(file.UncompressedSize64), 10))
+		w.Header().Set("Content-Type", contentType)
+		w.Header().Set("Content-Disposition", "attachment; filename=" + filepath.Base(file.Name))
+
+		reader, err := file.Open()
+		if err != nil {
+			helper.Fail500(w, err)
+		}
+		defer reader.Close()
+
+		// Copy file body
+		io.Copy(w, reader)
+	}, "")
+}

internal/artifacts/metadata.go

+package artifacts
+import (
+	"archive/zip"
+	"io"
+	"encoding/binary"
+	"encoding/json"
+)
+
+type metadata struct {
+	Modified uint16 `json:"modified"`
+	CRC      uint32 `json:"crc,omitempty"`
+	Size     uint64 `json:"size,omitempty"`
+	Zipped   uint64 `json:"zipped,omitempty"`
+	Comment  string `json:"comment,omitempty"`
+}
+
+func newMetadata(file *zip.File) metadata {
+	return metadata{
+		Modified: file.ModifiedDate,
+		CRC:      file.CRC32,
+		Size:     file.CompressedSize64,
+		Zipped:   file.UncompressedSize64,
+		Comment:  file.Comment,
+	}
+}
+
+func (m metadata) write(output io.Writer) error {
+	j, err := json.Marshal(m)
+	if err != nil {
+		return err
+	}
+	j = append(j, byte('\n'))
+	return writeBytes(output, j)
+}
+
+func generateZipMetadata(output io.Writer, archive *zip.Reader) error {
+	err := writeString(output, "GitLab Build Artifacts Metadata 0.0.1\n")
+	if err != nil {
+		return err
+	}
+	err = writeString(output, "{}")
+	if err != nil {
+		return err
+	}
+
+	for _, entry := range archive.File {
+		err = writeString(output, entry.Name)
+		if err != nil {
+			return err
+		}
+
+		err = newMetadata(entry).write(output)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func writeBytes(output io.Writer, data []byte) error {
+	err := binary.Write(output, binary.BigEndian, uint32(len(data)))
+	if err == nil {
+		_, err = output.Write(data)
+	}
+	return err
+}
+
+func writeString(output io.Writer, str string) error {
+	return writeBytes(output, []byte(str))
+}

internal/artifacts/upload_filter.go

+package artifacts
+
+import (
+	"mime/multipart"
+	"fmt"
+	"archive/zip"
+	"compress/gzip"
+)
+
+type artifactsFormFilter struct {
+}
+
+func (a *artifactsFormFilter) FilterFile(formName, fileName string, writer *multipart.Writer) error {
+	if formName != "file" {
+		return fmt.Errorf("Invalid form field: %q", formName)
+	}
+
+	archive, err := zip.OpenReader(fileName)
+	if err != nil {
+		// Ignore non-zip archives
+		return nil
+	}
+	defer archive.Close()
+
+	// TODO:
+	// we could create a temporary file and save to this file instead of writing to mulipart.Writer
+	// doing it like this is simpler, but puts more pressure on memory
+	metadataFile, err := writer.CreateFormFile("metadata", "metadata.gz")
+	if err != nil {
+		return err
+	}
+	defer writer.Close()
+
+	gz := gzip.NewWriter(metadataFile)
+	defer gz.Close()
+
+	err = generateZipMetadata(gz, &archive.Reader)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (a *artifactsFormFilter) FilterField(formName string, writer *multipart.Writer) error {
+	return nil
+}

internal/git/archive.go

@@ -23,6 +23,7 @@ import (
 func GetArchive(a *api.API) http.Handler {
 	return repoPreAuthorizeHandler(a, handleGetArchive)
 }
+
 func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
 	var format string
 	urlPath := r.URL.Path

internal/upload/uploads.go

@@ -3,7 +3,6 @@ package upload
 import (
 	"../helper"
 	"bytes"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -14,7 +13,12 @@ import (
 
 const tempPathHeader = "Gitlab-Workhorse-Temp-Path"
 
-func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, tempPath string) (cleanup func(), err error) {
+type MultipartFormFilter interface {
+	FilterFile(formName, fileName string, writer *multipart.Writer) error
+	FilterField(formName string, writer *multipart.Writer) error
+}
+
+func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, tempPath string, filter MultipartFormFilter) (cleanup func(), err error) {
 	// Create multipart reader
 	reader, err := r.MultipartReader()
 	if err != nil {
@@ -67,10 +71,18 @@ func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, te
 			files = append(files, file.Name())
 
 			_, err = io.Copy(file, p)
-			file.Close()
 			if err != nil {
 				return cleanup, err
 			}
+
+			file.Close()
+
+			if filter != nil {
+				err = filter.FilterFile(name, filename, writer)
+				if err != nil {
+					return cleanup, err
+				}
+			}
 		} else {
 			np, err := writer.CreatePart(p.Header)
 			if err != nil {
@@ -81,48 +93,46 @@ func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, te
 			if err != nil {
 				return cleanup, err
 			}
+
+			if filter != nil {
+				err = filter.FilterField(name, writer)
+				if err != nil {
+					return cleanup, err
+				}
+			}
 		}
 	}
 	return cleanup, nil
 }
 
-func handleFileUploads(h http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		tempPath := r.Header.Get(tempPathHeader)
-		if tempPath == "" {
-			helper.Fail500(w, errors.New("handleFileUploads: TempPath empty"))
-			return
-		}
-		r.Header.Del(tempPathHeader)
-
-		var body bytes.Buffer
-		writer := multipart.NewWriter(&body)
-		defer writer.Close()
+func HandleFileUploads(w http.ResponseWriter, r *http.Request, h http.Handler, tempPath string, filter MultipartFormFilter) {
+	var body bytes.Buffer
+	writer := multipart.NewWriter(&body)
+	defer writer.Close()
 
-		// Rewrite multipart form data
-		cleanup, err := rewriteFormFilesFromMultipart(r, writer, tempPath)
-		if err != nil {
-			if err == http.ErrNotMultipart {
-				h.ServeHTTP(w, r)
-			} else {
-				helper.Fail500(w, fmt.Errorf("handleFileUploads: extract files from multipart: %v", err))
-			}
-			return
+	// Rewrite multipart form data
+	cleanup, err := rewriteFormFilesFromMultipart(r, writer, tempPath, filter)
+	if err != nil {
+		if err == http.ErrNotMultipart {
+			h.ServeHTTP(w, r)
+		} else {
+			helper.Fail500(w, fmt.Errorf("handleFileUploads: extract files from multipart: %v", err))
 		}
+		return
+	}
 
-		if cleanup != nil {
-			defer cleanup()
-		}
+	if cleanup != nil {
+		defer cleanup()
+	}
 
-		// Close writer
-		writer.Close()
+	// Close writer
+	writer.Close()
 
-		// Hijack the request
-		r.Body = ioutil.NopCloser(&body)
-		r.ContentLength = int64(body.Len())
-		r.Header.Set("Content-Type", writer.FormDataContentType())
+	// Hijack the request
+	r.Body = ioutil.NopCloser(&body)
+	r.ContentLength = int64(body.Len())
+	r.Header.Set("Content-Type", writer.FormDataContentType())
 
-		// Proxy the request
-		h.ServeHTTP(w, r)
-	})
+	// Proxy the request
+	h.ServeHTTP(w, r)
 }

internal/upstream/routes.go

@@ -2,11 +2,11 @@ package upstream
 
 import (
 	apipkg "../api"
+	"../artifacts"
 	"../git"
 	"../lfs"
 	proxypkg "../proxy"
 	"../staticpages"
-	"../upload"
 	"net/http"
 	"regexp"
 )
@@ -23,7 +23,7 @@ const gitProjectPattern = `^/[^/]+/[^/]+\.git/`
 const apiPattern = `^/api/`
 
 // A project ID in an API request is either a number or two strings 'namespace/project'
-const projectsAPIPattern = `^/api/v3/projects/((\d+)|([^/]+/[^/]+))/`
+const projectsAPIPattern = `^/api/v3/projects/(\d+)|([^/]+/[^/]+)/`
 const ciAPIPattern = `^/ci/api/`
 
 // Routing table
@@ -64,8 +64,9 @@ func (u *Upstream) configureRoutes() {
 		route{"GET", regexp.MustCompile(projectsAPIPattern + `repository/archive.tar.gz\z`), git.GetArchive(api)},
 		route{"GET", regexp.MustCompile(projectsAPIPattern + `repository/archive.tar.bz2\z`), git.GetArchive(api)},
 
-		// CI Artifacts API
-		route{"POST", regexp.MustCompile(ciAPIPattern + `v1/builds/[0-9]+/artifacts\z`), contentEncodingHandler(upload.Artifacts(api, proxy))},
+		// CI Artifacts
+		route{"GET", regexp.MustCompile(projectPattern + `builds/[0-9]+/file/`), contentEncodingHandler(artifacts.DownloadArtifact(api))},
+		route{"POST", regexp.MustCompile(ciAPIPattern + `v1/builds/[0-9]+/artifacts\z`), contentEncodingHandler(artifacts.UploadArtifacts(api, proxy))},
 
 		// Explicitly proxy API requests
 		route{"", regexp.MustCompile(apiPattern), proxy},