Commit adada74d authored by Jacob Vosmaer's avatar Jacob Vosmaer

Merge branch 'artifacts-metadata' into 'master'

Add support for new artifacts features

It is proof of concept implementation that misses following:

- [x] Saving metadata to file, and passing that file to rails,
- [x] Making it look nice,
- [x] Have `refactor-upstream` merged: https://gitlab.com/gitlab-org/gitlab-workhorse/merge_requests/20
- [x] Add tests for metadata and single file downloads,
- [ ] Spawning separate process to verify zip archive **ONHOLD**: https://gitlab.com/gitlab-org/gitlab-workhorse/issues/17,
- [ ] Spawning separate process to download file from zip archive **ONHOLD**: https://gitlab.com/gitlab-org/gitlab-workhorse/issues/17


See merge request !28
parents 3bbb1be7 9cf03a66
...@@ -58,6 +58,10 @@ type Response struct { ...@@ -58,6 +58,10 @@ type Response struct {
// TmpPath is the path where we should store temporary files // TmpPath is the path where we should store temporary files
// This is set by authorization middleware // This is set by authorization middleware
TempPath string TempPath string
// Archive is the path where the artifacts archive is stored
Archive string `json:"archive"`
// Entry is a filename inside the archive point to file that needs to be extracted
Entry string `json:"entry"`
} }
// singleJoiningSlash is taken from reverseproxy.go:NewSingleHostReverseProxy // singleJoiningSlash is taken from reverseproxy.go:NewSingleHostReverseProxy
......
package artifacts
import (
"../api"
"../helper"
"archive/zip"
"encoding/base64"
"errors"
"fmt"
"io"
"mime"
"net/http"
"os"
"path/filepath"
"strconv"
)
func decodeFileEntry(entry string) (string, error) {
decoded, err := base64.StdEncoding.DecodeString(entry)
if err != nil {
return "", err
}
return string(decoded), nil
}
func detectFileContentType(fileName string) string {
contentType := mime.TypeByExtension(filepath.Ext(fileName))
if contentType == "" {
contentType = "application/octet-stream"
}
return contentType
}
func findFileInZip(fileName string, archive *zip.Reader) *zip.File {
for _, file := range archive.File {
if file.Name == fileName {
return file
}
}
return nil
}
func unpackFileFromZip(archiveFileName, fileName string, headers http.Header, output io.Writer) error {
archive, err := zip.OpenReader(archiveFileName)
if err != nil {
return err
}
defer archive.Close()
file := findFileInZip(fileName, &archive.Reader)
if file == nil {
return os.ErrNotExist
}
// Start decompressing the file
reader, err := file.Open()
if err != nil {
return err
}
defer reader.Close()
basename := filepath.Base(fileName)
// Write http headers about the file
headers.Set("Content-Length", strconv.FormatInt(int64(file.UncompressedSize64), 10))
headers.Set("Content-Type", detectFileContentType(file.Name))
headers.Set("Content-Disposition", "attachment; filename=\""+escapeQuotes(basename)+"\"")
// Copy file body to client
_, err = io.Copy(output, reader)
return err
}
// Artifacts downloader doesn't support ranges when downloading a single file
func DownloadArtifact(myAPI *api.API) http.Handler {
return myAPI.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) {
if a.Archive == "" || a.Entry == "" {
helper.Fail500(w, errors.New("DownloadArtifact: Archive or Path is empty"))
return
}
fileName, err := decodeFileEntry(a.Entry)
if err != nil {
helper.Fail500(w, err)
return
}
err = unpackFileFromZip(a.Archive, fileName, w.Header(), w)
if os.IsNotExist(err) {
http.NotFound(w, r)
return
} else if err != nil {
helper.Fail500(w, fmt.Errorf("DownloadArtifact: %v", err))
}
}, "")
}
package artifacts
import (
"../api"
"../helper"
"../testhelper"
"archive/zip"
"encoding/base64"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
"testing"
)
func testArtifactDownloadServer(t *testing.T, archive string, entry string) *httptest.Server {
mux := http.NewServeMux()
mux.HandleFunc("/url/path", func(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
t.Fatal("Expected GET request")
}
w.Header().Set("Content-Type", "application/json")
data, err := json.Marshal(&api.Response{
Archive: archive,
Entry: base64.StdEncoding.EncodeToString([]byte(entry)),
})
if err != nil {
t.Fatal(err)
}
w.Write(data)
})
return testhelper.TestServerWithHandler(nil, mux.ServeHTTP)
}
func testDownloadArtifact(t *testing.T, ts *httptest.Server) *httptest.ResponseRecorder {
httpRequest, err := http.NewRequest("GET", ts.URL+"/url/path", nil)
if err != nil {
t.Fatal(err)
}
response := httptest.NewRecorder()
apiClient := api.NewAPI(helper.URLMustParse(ts.URL), "123", nil)
DownloadArtifact(apiClient).ServeHTTP(response, httpRequest)
return response
}
func TestDownloadingFromValidArchive(t *testing.T) {
tempFile, err := ioutil.TempFile("", "uploads")
if err != nil {
t.Fatal(err)
}
defer tempFile.Close()
defer os.Remove(tempFile.Name())
archive := zip.NewWriter(tempFile)
defer archive.Close()
fileInArchive, err := archive.Create("test.txt")
if err != nil {
t.Fatal(err)
}
fmt.Fprint(fileInArchive, "testtest")
archive.Close()
ts := testArtifactDownloadServer(t, tempFile.Name(), "test.txt")
defer ts.Close()
response := testDownloadArtifact(t, ts)
testhelper.AssertResponseCode(t, response, 200)
testhelper.AssertResponseHeader(t, response,
"Content-Type",
"text/plain; charset=utf-8")
testhelper.AssertResponseHeader(t, response,
"Content-Disposition",
"attachment; filename=\"test.txt\"")
testhelper.AssertResponseBody(t, response, "testtest")
}
func TestDownloadingNonExistingFile(t *testing.T) {
tempFile, err := ioutil.TempFile("", "uploads")
if err != nil {
t.Fatal(err)
}
defer tempFile.Close()
defer os.Remove(tempFile.Name())
archive := zip.NewWriter(tempFile)
defer archive.Close()
archive.Close()
ts := testArtifactDownloadServer(t, tempFile.Name(), "test")
defer ts.Close()
response := testDownloadArtifact(t, ts)
testhelper.AssertResponseCode(t, response, 404)
}
func TestDownloadingFromInvalidArchive(t *testing.T) {
ts := testArtifactDownloadServer(t, "path/to/non/existing/file", "test")
defer ts.Close()
response := testDownloadArtifact(t, ts)
testhelper.AssertResponseCode(t, response, 404)
}
func TestIncompleteApiResponse(t *testing.T) {
ts := testArtifactDownloadServer(t, "", "")
defer ts.Close()
response := testDownloadArtifact(t, ts)
testhelper.AssertResponseCode(t, response, 500)
}
package artifacts
import (
"../api"
"../helper"
"../upload"
"errors"
"fmt"
"io/ioutil"
"mime/multipart"
"net/http"
"os"
)
type artifactsUploadProcessor struct {
TempPath string
metadataFile string
}
func (a *artifactsUploadProcessor) ProcessFile(formName, fileName string, writer *multipart.Writer) error {
// ProcessFile for artifacts requires file form-data field name to eq `file`
if formName != "file" {
return fmt.Errorf("Invalid form field: %q", formName)
}
if a.metadataFile != "" {
return fmt.Errorf("Artifacts request contains more than one file!")
}
// Create temporary file for metadata and store it's path
tempFile, err := ioutil.TempFile(a.TempPath, "metadata_")
if err != nil {
return err
}
defer tempFile.Close()
a.metadataFile = tempFile.Name()
// Generate metadata and save to file
err = generateZipMetadataFromFile(fileName, tempFile)
if err == os.ErrInvalid {
return nil
} else if err != nil {
return err
}
// Pass metadata file path to Rails
writer.WriteField("metadata.path", a.metadataFile)
writer.WriteField("metadata.name", "metadata.gz")
return nil
}
func (a *artifactsUploadProcessor) ProcessField(formName string, writer *multipart.Writer) error {
return nil
}
func (a *artifactsUploadProcessor) Cleanup() {
if a.metadataFile != "" {
os.Remove(a.metadataFile)
}
}
func UploadArtifacts(myAPI *api.API, h http.Handler) http.Handler {
return myAPI.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) {
if a.TempPath == "" {
helper.Fail500(w, errors.New("UploadArtifacts: TempPath is empty"))
return
}
mg := &artifactsUploadProcessor{TempPath: a.TempPath}
defer mg.Cleanup()
upload.HandleFileUploads(w, r, h, a.TempPath, mg)
}, "/authorize")
}
package artifacts
import (
"../api"
"../helper"
"../proxy"
"../testhelper"
"archive/zip"
"bytes"
"compress/gzip"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"mime/multipart"
"net/http"
"net/http/httptest"
"os"
"testing"
)
func testArtifactsUploadServer(t *testing.T, tempPath string) *httptest.Server {
mux := http.NewServeMux()
mux.HandleFunc("/url/path/authorize", func(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
t.Fatal("Expected POST request")
}
w.Header().Set("Content-Type", "application/json")
data, err := json.Marshal(&api.Response{
TempPath: tempPath,
})
if err != nil {
t.Fatal("Expected to marshal")
}
w.Write(data)
})
mux.HandleFunc("/url/path", func(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
t.Fatal("Expected POST request")
}
if r.FormValue("file.path") == "" {
w.WriteHeader(501)
return
}
if r.FormValue("metadata.path") == "" {
w.WriteHeader(502)
return
}
_, err := ioutil.ReadFile(r.FormValue("file.path"))
if err != nil {
w.WriteHeader(404)
return
}
metadata, err := ioutil.ReadFile(r.FormValue("metadata.path"))
if err != nil {
w.WriteHeader(404)
return
}
gz, err := gzip.NewReader(bytes.NewReader(metadata))
if err != nil {
w.WriteHeader(405)
return
}
defer gz.Close()
metadata, err = ioutil.ReadAll(gz)
if err != nil {
w.WriteHeader(404)
return
}
if !bytes.HasPrefix(metadata, []byte(metadataHeaderPrefix+metadataHeader)) {
w.WriteHeader(400)
return
}
w.WriteHeader(200)
})
return testhelper.TestServerWithHandler(nil, mux.ServeHTTP)
}
func testUploadArtifacts(contentType string, body io.Reader, t *testing.T, ts *httptest.Server) *httptest.ResponseRecorder {
httpRequest, err := http.NewRequest("POST", ts.URL+"/url/path", body)
if err != nil {
t.Fatal(err)
}
httpRequest.Header.Set("Content-Type", contentType)
response := httptest.NewRecorder()
apiClient := api.NewAPI(helper.URLMustParse(ts.URL), "123", nil)
proxyClient := proxy.NewProxy(helper.URLMustParse(ts.URL), "123", nil)
UploadArtifacts(apiClient, proxyClient).ServeHTTP(response, httpRequest)
return response
}
func TestUploadHandlerAddingMetadata(t *testing.T) {
tempPath, err := ioutil.TempDir("", "uploads")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempPath)
ts := testArtifactsUploadServer(t, tempPath)
defer ts.Close()
var buffer bytes.Buffer
writer := multipart.NewWriter(&buffer)
file, err := writer.CreateFormFile("file", "my.file")
if err != nil {
t.Fatal(err)
}
archive := zip.NewWriter(file)
defer archive.Close()
fileInArchive, err := archive.Create("test.file")
if err != nil {
t.Fatal(err)
}
fmt.Fprint(fileInArchive, "test")
archive.Close()
writer.Close()
response := testUploadArtifacts(writer.FormDataContentType(), &buffer, t, ts)
testhelper.AssertResponseCode(t, response, 200)
}
func TestUploadHandlerForUnsupportedArchive(t *testing.T) {
tempPath, err := ioutil.TempDir("", "uploads")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempPath)
ts := testArtifactsUploadServer(t, tempPath)
defer ts.Close()
var buffer bytes.Buffer
writer := multipart.NewWriter(&buffer)
file, err := writer.CreateFormFile("file", "my.file")
if err != nil {
t.Fatal(err)
}
fmt.Fprint(file, "test")
writer.Close()
response := testUploadArtifacts(writer.FormDataContentType(), &buffer, t, ts)
// 502 is a custom response code from the mock server in testUploadArtifacts
testhelper.AssertResponseCode(t, response, 502)
}
func TestUploadFormProcessing(t *testing.T) {
tempPath, err := ioutil.TempDir("", "uploads")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempPath)
ts := testArtifactsUploadServer(t, tempPath)
defer ts.Close()
var buffer bytes.Buffer
writer := multipart.NewWriter(&buffer)
file, err := writer.CreateFormFile("metadata", "my.file")
if err != nil {
t.Fatal(err)
}
fmt.Fprint(file, "test")
writer.Close()
response := testUploadArtifacts(writer.FormDataContentType(), &buffer, t, ts)
testhelper.AssertResponseCode(t, response, 500)
}
package artifacts
import "strings"
// taken from mime/multipart/writer.go
var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"")
func escapeQuotes(s string) string {
return quoteEscaper.Replace(s)
}
package artifacts
import (
"archive/zip"
"compress/gzip"
"encoding/binary"
"encoding/json"
"io"
"os"
"strconv"
)
type metadata struct {
Modified int64 `json:"modified"`
Mode string `json:"mode"`
CRC uint32 `json:"crc,omitempty"`
Size uint64 `json:"size,omitempty"`
Zipped uint64 `json:"zipped,omitempty"`
Comment string `json:"comment,omitempty"`
}
const metadataHeaderPrefix = "\x00\x00\x00&" // length of string below, encoded properly
const metadataHeader = "GitLab Build Artifacts Metadata 0.0.2\n"
func newMetadata(file *zip.File) metadata {
return metadata{
Modified: file.ModTime().Unix(),
Mode: strconv.FormatUint(uint64(file.Mode().Perm()), 8),
CRC: file.CRC32,
Size: file.UncompressedSize64,
Zipped: file.CompressedSize64,
Comment: file.Comment,
}
}
func (m metadata) writeEncoded(output io.Writer) error {
j, err := json.Marshal(m)
if err != nil {
return err
}
j = append(j, byte('\n'))
return writeBytes(output, j)
}
func writeZipEntryMetadata(output io.Writer, entry *zip.File) error {
err := writeString(output, entry.Name)
if err != nil {
return err
}
err = newMetadata(entry).writeEncoded(output)
if err != nil {
return err
}
return nil
}
func generateZipMetadata(output io.Writer, archive *zip.Reader) error {
err := writeString(output, metadataHeader)
if err != nil {
return err
}
// Write empty error string
err = writeString(output, "{}")
if err != nil {
return err
}
// Write all files
for _, entry := range archive.File {
err = writeZipEntryMetadata(output, entry)
if err != nil {
return err
}
}
return nil
}
func generateZipMetadataFromFile(fileName string, w io.Writer) error {
archive, err := zip.OpenReader(fileName)
if err != nil {
// Ignore non-zip archives
return os.ErrInvalid
}
defer archive.Close()
gz := gzip.NewWriter(w)
defer gz.Close()
return generateZipMetadata(gz, &archive.Reader)
}
func writeBytes(output io.Writer, data []byte) error {
err := binary.Write(output, binary.BigEndian, uint32(len(data)))
if err == nil {
_, err = output.Write(data)
}
return err
}
func writeString(output io.Writer, str string) error {
return writeBytes(output, []byte(str))
}
...@@ -23,6 +23,7 @@ import ( ...@@ -23,6 +23,7 @@ import (
func GetArchive(a *api.API) http.Handler { func GetArchive(a *api.API) http.Handler {
return repoPreAuthorizeHandler(a, handleGetArchive) return repoPreAuthorizeHandler(a, handleGetArchive)
} }
func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) { func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
var format string var format string
urlPath := r.URL.Path urlPath := r.URL.Path
......
package upload
import (
"../api"
"net/http"
)
func Artifacts(myAPI *api.API, h http.Handler) http.Handler {
return myAPI.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) {
r.Header.Set(tempPathHeader, a.TempPath)
handleFileUploads(h).ServeHTTP(w, r)
}, "/authorize")
}
...@@ -3,7 +3,6 @@ package upload ...@@ -3,7 +3,6 @@ package upload
import ( import (
"../helper" "../helper"
"bytes" "bytes"
"errors"
"fmt" "fmt"
"io" "io"
"io/ioutil" "io/ioutil"
...@@ -12,9 +11,12 @@ import ( ...@@ -12,9 +11,12 @@ import (
"os" "os"
) )
const tempPathHeader = "Gitlab-Workhorse-Temp-Path" type MultipartFormProcessor interface {
ProcessFile(formName, fileName string, writer *multipart.Writer) error
ProcessField(formName string, writer *multipart.Writer) error
}
func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, tempPath string) (cleanup func(), err error) { func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, tempPath string, filter MultipartFormProcessor) (cleanup func(), err error) {
// Create multipart reader // Create multipart reader
reader, err := r.MultipartReader() reader, err := r.MultipartReader()
if err != nil { if err != nil {
...@@ -67,10 +69,15 @@ func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, te ...@@ -67,10 +69,15 @@ func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, te
files = append(files, file.Name()) files = append(files, file.Name())
_, err = io.Copy(file, p) _, err = io.Copy(file, p)
file.Close()
if err != nil { if err != nil {
return cleanup, err return cleanup, err
} }
file.Close()
if err := filter.ProcessFile(name, file.Name(), writer); err != nil {
return cleanup, err
}
} else { } else {
np, err := writer.CreatePart(p.Header) np, err := writer.CreatePart(p.Header)
if err != nil { if err != nil {
...@@ -81,48 +88,48 @@ func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, te ...@@ -81,48 +88,48 @@ func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, te
if err != nil { if err != nil {
return cleanup, err return cleanup, err
} }
if err := filter.ProcessField(name, writer); err != nil {
return cleanup, err
}
} }
} }
return cleanup, nil return cleanup, nil
} }
func handleFileUploads(h http.Handler) http.Handler { func HandleFileUploads(w http.ResponseWriter, r *http.Request, h http.Handler, tempPath string, filter MultipartFormProcessor) {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if tempPath == "" {
tempPath := r.Header.Get(tempPathHeader) helper.Fail500(w, fmt.Errorf("handleFileUploads: temporary path not defined"))
if tempPath == "" { return
helper.Fail500(w, errors.New("handleFileUploads: TempPath empty")) }
return
}
r.Header.Del(tempPathHeader)
var body bytes.Buffer var body bytes.Buffer
writer := multipart.NewWriter(&body) writer := multipart.NewWriter(&body)
defer writer.Close() defer writer.Close()
// Rewrite multipart form data // Rewrite multipart form data
cleanup, err := rewriteFormFilesFromMultipart(r, writer, tempPath) cleanup, err := rewriteFormFilesFromMultipart(r, writer, tempPath, filter)
if err != nil { if err != nil {
if err == http.ErrNotMultipart { if err == http.ErrNotMultipart {
h.ServeHTTP(w, r) h.ServeHTTP(w, r)
} else { } else {
helper.Fail500(w, fmt.Errorf("handleFileUploads: extract files from multipart: %v", err)) helper.Fail500(w, fmt.Errorf("handleFileUploads: extract files from multipart: %v", err))
}
return
} }
return
}
if cleanup != nil { if cleanup != nil {
defer cleanup() defer cleanup()
} }
// Close writer // Close writer
writer.Close() writer.Close()
// Hijack the request // Hijack the request
r.Body = ioutil.NopCloser(&body) r.Body = ioutil.NopCloser(&body)
r.ContentLength = int64(body.Len()) r.ContentLength = int64(body.Len())
r.Header.Set("Content-Type", writer.FormDataContentType()) r.Header.Set("Content-Type", writer.FormDataContentType())
// Proxy the request // Proxy the request
h.ServeHTTP(w, r) h.ServeHTTP(w, r)
})
} }
...@@ -5,6 +5,7 @@ import ( ...@@ -5,6 +5,7 @@ import (
"../proxy" "../proxy"
"../testhelper" "../testhelper"
"bytes" "bytes"
"errors"
"fmt" "fmt"
"io" "io"
"io/ioutil" "io/ioutil"
...@@ -19,10 +20,27 @@ import ( ...@@ -19,10 +20,27 @@ import (
var nilHandler = http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}) var nilHandler = http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})
type testFormProcessor struct {
}
func (a *testFormProcessor) ProcessFile(formName, fileName string, writer *multipart.Writer) error {
if formName != "file" && fileName != "my.file" {
return errors.New("illegal file")
}
return nil
}
func (a *testFormProcessor) ProcessField(formName string, writer *multipart.Writer) error {
if formName != "token" {
return errors.New("illegal field")
}
return nil
}
func TestUploadTempPathRequirement(t *testing.T) { func TestUploadTempPathRequirement(t *testing.T) {
response := httptest.NewRecorder() response := httptest.NewRecorder()
request := &http.Request{} request := &http.Request{}
handleFileUploads(nilHandler).ServeHTTP(response, request) HandleFileUploads(response, request, nilHandler, "", nil)
testhelper.AssertResponseCode(t, response, 500) testhelper.AssertResponseCode(t, response, 500)
} }
...@@ -56,9 +74,8 @@ func TestUploadHandlerForwardingRawData(t *testing.T) { ...@@ -56,9 +74,8 @@ func TestUploadHandlerForwardingRawData(t *testing.T) {
response := httptest.NewRecorder() response := httptest.NewRecorder()
httpRequest.Header.Set(tempPathHeader, tempPath) handler := proxy.NewProxy(helper.URLMustParse(ts.URL), "123", nil)
HandleFileUploads(response, httpRequest, handler, tempPath, nil)
handleFileUploads(proxy.NewProxy(helper.URLMustParse(ts.URL), "123", nil)).ServeHTTP(response, httpRequest)
testhelper.AssertResponseCode(t, response, 202) testhelper.AssertResponseCode(t, response, 202)
if response.Body.String() != "RESPONSE" { if response.Body.String() != "RESPONSE" {
t.Fatal("Expected RESPONSE in response body") t.Fatal("Expected RESPONSE in response body")
...@@ -129,13 +146,65 @@ func TestUploadHandlerRewritingMultiPartData(t *testing.T) { ...@@ -129,13 +146,65 @@ func TestUploadHandlerRewritingMultiPartData(t *testing.T) {
httpRequest.Body = ioutil.NopCloser(&buffer) httpRequest.Body = ioutil.NopCloser(&buffer)
httpRequest.ContentLength = int64(buffer.Len()) httpRequest.ContentLength = int64(buffer.Len())
httpRequest.Header.Set("Content-Type", writer.FormDataContentType()) httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
httpRequest.Header.Set(tempPathHeader, tempPath)
response := httptest.NewRecorder() response := httptest.NewRecorder()
handleFileUploads(proxy.NewProxy(helper.URLMustParse(ts.URL), "123", nil)).ServeHTTP(response, httpRequest) handler := proxy.NewProxy(helper.URLMustParse(ts.URL), "123", nil)
HandleFileUploads(response, httpRequest, handler, tempPath, &testFormProcessor{})
testhelper.AssertResponseCode(t, response, 202) testhelper.AssertResponseCode(t, response, 202)
if _, err := os.Stat(filePath); !os.IsNotExist(err) { if _, err := os.Stat(filePath); !os.IsNotExist(err) {
t.Fatal("expected the file to be deleted") t.Fatal("expected the file to be deleted")
} }
} }
func TestUploadProcessingField(t *testing.T) {
tempPath, err := ioutil.TempDir("", "uploads")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempPath)
var buffer bytes.Buffer
writer := multipart.NewWriter(&buffer)
writer.WriteField("token2", "test")
writer.Close()
httpRequest, err := http.NewRequest("PUT", "/url/path", &buffer)
if err != nil {
t.Fatal(err)
}
httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
response := httptest.NewRecorder()
HandleFileUploads(response, httpRequest, nilHandler, tempPath, &testFormProcessor{})
testhelper.AssertResponseCode(t, response, 500)
}
func TestUploadProcessingFile(t *testing.T) {
tempPath, err := ioutil.TempDir("", "uploads")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempPath)
var buffer bytes.Buffer
writer := multipart.NewWriter(&buffer)
file, err := writer.CreateFormFile("file2", "my.file")
if err != nil {
t.Fatal(err)
}
fmt.Fprint(file, "test")
writer.Close()
httpRequest, err := http.NewRequest("PUT", "/url/path", &buffer)
if err != nil {
t.Fatal(err)
}
httpRequest.Header.Set("Content-Type", writer.FormDataContentType())
response := httptest.NewRecorder()
HandleFileUploads(response, httpRequest, nilHandler, tempPath, &testFormProcessor{})
testhelper.AssertResponseCode(t, response, 500)
}
...@@ -2,11 +2,11 @@ package upstream ...@@ -2,11 +2,11 @@ package upstream
import ( import (
apipkg "../api" apipkg "../api"
"../artifacts"
"../git" "../git"
"../lfs" "../lfs"
proxypkg "../proxy" proxypkg "../proxy"
"../staticpages" "../staticpages"
"../upload"
"net/http" "net/http"
"regexp" "regexp"
) )
...@@ -64,8 +64,9 @@ func (u *Upstream) configureRoutes() { ...@@ -64,8 +64,9 @@ func (u *Upstream) configureRoutes() {
route{"GET", regexp.MustCompile(projectsAPIPattern + `repository/archive.tar.gz\z`), git.GetArchive(api)}, route{"GET", regexp.MustCompile(projectsAPIPattern + `repository/archive.tar.gz\z`), git.GetArchive(api)},
route{"GET", regexp.MustCompile(projectsAPIPattern + `repository/archive.tar.bz2\z`), git.GetArchive(api)}, route{"GET", regexp.MustCompile(projectsAPIPattern + `repository/archive.tar.bz2\z`), git.GetArchive(api)},
// CI Artifacts API // CI Artifacts
route{"POST", regexp.MustCompile(ciAPIPattern + `v1/builds/[0-9]+/artifacts\z`), contentEncodingHandler(upload.Artifacts(api, proxy))}, route{"GET", regexp.MustCompile(projectPattern + `builds/[0-9]+/artifacts/file/`), contentEncodingHandler(artifacts.DownloadArtifact(api))},
route{"POST", regexp.MustCompile(ciAPIPattern + `v1/builds/[0-9]+/artifacts\z`), contentEncodingHandler(artifacts.UploadArtifacts(api, proxy))},
// Explicitly proxy API requests // Explicitly proxy API requests
route{"", regexp.MustCompile(apiPattern), proxy}, route{"", regexp.MustCompile(apiPattern), proxy},
......
...@@ -6,6 +6,7 @@ import ( ...@@ -6,6 +6,7 @@ import (
"./internal/testhelper" "./internal/testhelper"
"./internal/upstream" "./internal/upstream"
"bytes" "bytes"
"encoding/base64"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
...@@ -488,6 +489,40 @@ func TestArtifactsUpload(t *testing.T) { ...@@ -488,6 +489,40 @@ func TestArtifactsUpload(t *testing.T) {
} }
} }
func TestArtifactsGetSingleFile(t *testing.T) {
// We manually created this zip file in the gitlab-workhorse Git repository
archivePath := `testdata/artifacts-archive.zip`
fileName := "myfile"
fileContents := "MY FILE"
resourcePath := `/namespace/project/builds/123/artifacts/file/` + fileName
ts := testhelper.TestServerWithHandler(regexp.MustCompile(`\A`+resourcePath+`\z`), func(w http.ResponseWriter, r *http.Request) {
encodedFilename := base64.StdEncoding.EncodeToString([]byte(fileName))
if _, err := fmt.Fprintf(w, `{"Archive":"%s","Entry":"%s"}`, archivePath, encodedFilename); err != nil {
t.Fatal(err)
}
return
})
defer ts.Close()
ws := startWorkhorseServer(ts.URL)
defer ws.Close()
resp, err := http.Get(ws.URL + resourcePath)
if err != nil {
t.Error(err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
t.Errorf("GET %q: expected 200, got %d", resourcePath, resp.StatusCode)
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatal(err)
}
if string(body) != fileContents {
t.Fatalf("Expected file contents %q, got %q", fileContents, body)
}
}
func setupStaticFile(fpath, content string) error { func setupStaticFile(fpath, content string) error {
cwd, err := os.Getwd() cwd, err := os.Getwd()
if err != nil { if err != nil {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment