Commit c5e8ad40 authored by Jacob Vosmaer's avatar Jacob Vosmaer

Merge branch 'lfs_support' into 'master'

LFS storage api support

Part of gitlab-org/gitlab-ce#2955

See merge request !3
parents 84eb6e76 fb643f90
PREFIX=/usr/local PREFIX=/usr/local
VERSION=$(shell git describe)-$(shell date -u +%Y%m%d.%H%M%S) VERSION=$(shell git describe)-$(shell date -u +%Y%m%d.%H%M%S)
gitlab-workhorse: main.go upstream.go archive.go git-http.go helpers.go xsendfile.go authorization.go gitlab-workhorse: main.go upstream.go archive.go git-http.go helpers.go xsendfile.go authorization.go lfs.go
go build -ldflags "-X main.Version ${VERSION}" -o gitlab-workhorse go build -ldflags "-X main.Version ${VERSION}" -o gitlab-workhorse
install: gitlab-workhorse install: gitlab-workhorse
......
/*
In this file we handle git lfs objects downloads and uploads
*/
package main
import (
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"io/ioutil"
"net/http"
"os"
"path/filepath"
)
var (
errHashMismatch = errors.New("Content hash does not match OID")
errSizeMismatch = errors.New("Content size does not match")
)
func lfsAuthorizeHandler(handleFunc serviceHandleFunc) serviceHandleFunc {
return preAuthorizeHandler(func(w http.ResponseWriter, r *gitRequest) {
if r.StoreLFSPath == "" {
fail500(w, "lfsAuthorizeHandler", errors.New("Don't know where to store object, no store path specified."))
return
}
if r.LfsOid == "" {
fail500(w, "lfsAuthorizeHandler", errors.New("Lfs object oid not specified."))
return
}
if r.LfsSize == 0 {
fail500(w, "lfsAuthorizeHandler", errors.New("Lfs object size not specified."))
return
}
if err := os.MkdirAll(r.StoreLFSPath, 0700); err != nil {
fail500(w, "Couldn't create directory for storing LFS tmp objects.", err)
return
}
handleFunc(w, r)
}, "/authorize")
}
func handleStoreLfsObject(w http.ResponseWriter, r *gitRequest) {
var body io.ReadCloser
body = r.Body
defer body.Close()
file, err := ioutil.TempFile(r.StoreLFSPath, r.LfsOid)
if err != nil {
fail500(w, "Couldn't open tmp file for writing.", err)
return
}
defer os.Remove(file.Name())
defer file.Close()
hash := sha256.New()
hw := io.MultiWriter(hash, file)
written, err := io.Copy(hw, body)
if err != nil {
fail500(w, "Failed to save received LFS object.", err)
return
}
file.Close()
if written != r.LfsSize {
fail500(w, "Inconsistent size: ", errSizeMismatch)
return
}
shaStr := hex.EncodeToString(hash.Sum(nil))
if shaStr != r.LfsOid {
fail500(w, "Inconsistent size: ", errSizeMismatch)
return
}
r.Header.Set("X-GitLab-Lfs-Tmp", filepath.Base(file.Name()))
authReq, err := r.u.newUpstreamRequest(r.Request, nil, "")
if err != nil {
fail500(w, "newUpstreamRequestlfsCallback", err)
return
}
authResponse, err := r.u.httpClient.Do(authReq)
if err != nil {
fail500(w, "doRequestlfsCallback", err)
return
}
defer authResponse.Body.Close()
return
}
...@@ -241,69 +241,18 @@ func TestDownloadCacheCreate(t *testing.T) { ...@@ -241,69 +241,18 @@ func TestDownloadCacheCreate(t *testing.T) {
func TestAllowedXSendfileDownload(t *testing.T) { func TestAllowedXSendfileDownload(t *testing.T) {
contentFilename := "my-content" contentFilename := "my-content"
contentPath := path.Join(cacheDir, contentFilename) url := fmt.Sprintf("http://%s/foo/uploads/bar", servAddr)
prepareDownloadDir(t) prepareDownloadDir(t)
// Prepare test server and backend allowedXSendfileDownload(t, contentFilename, url)
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if xSendfileType := r.Header.Get("X-Sendfile-Type"); xSendfileType != "X-Sendfile" {
t.Fatalf(`X-Sendfile-Type want "X-Sendfile" got %q`, xSendfileType)
}
w.Header().Set("X-Sendfile", contentPath)
w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, contentFilename))
w.WriteHeader(200)
}))
defer ts.Close()
defer cleanUpProcessGroup(startServerOrFail(t, ts))
if err := os.MkdirAll(cacheDir, 0755); err != nil {
t.Fatal(err)
}
contentBytes := []byte{'c', 'o', 'n', 't', 'e', 'n', 't'}
if err := ioutil.WriteFile(contentPath, contentBytes, 0644); err != nil {
t.Fatal(err)
}
downloadCmd := exec.Command("curl", "-J", "-O", fmt.Sprintf("http://%s/foo/uploads/bar", servAddr))
downloadCmd.Dir = scratchDir
runOrFail(t, downloadCmd)
actual, err := ioutil.ReadFile(path.Join(scratchDir, contentFilename))
if err != nil {
t.Fatal(err)
}
if bytes.Compare(actual, contentBytes) != 0 {
t.Fatal("Unexpected file contents in download")
}
} }
func TestDeniedXSendfileDownload(t *testing.T) { func TestDeniedXSendfileDownload(t *testing.T) {
contentFilename := "my-content" contentFilename := "my-content"
url := fmt.Sprintf("http://%s/foo/uploads/bar", servAddr)
prepareDownloadDir(t) prepareDownloadDir(t)
// Prepare test server and backend deniedXSendfileDownload(t, contentFilename, url)
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if xSendfileType := r.Header.Get("X-Sendfile-Type"); xSendfileType != "X-Sendfile" {
t.Fatalf(`X-Sendfile-Type want "X-Sendfile" got %q`, xSendfileType)
}
w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, contentFilename))
w.WriteHeader(200)
fmt.Fprint(w, "Denied")
}))
defer ts.Close()
defer cleanUpProcessGroup(startServerOrFail(t, ts))
downloadCmd := exec.Command("curl", "-J", "-O", fmt.Sprintf("http://%s/foo/uploads/bar", servAddr))
downloadCmd.Dir = scratchDir
runOrFail(t, downloadCmd)
actual, err := ioutil.ReadFile(path.Join(scratchDir, contentFilename))
if err != nil {
t.Fatal(err)
}
if bytes.Compare(actual, []byte{'D', 'e', 'n', 'i', 'e', 'd'}) != 0 {
t.Fatal("Unexpected file contents in download")
}
} }
func prepareDownloadDir(t *testing.T) { func prepareDownloadDir(t *testing.T) {
...@@ -337,7 +286,7 @@ func testAuthServer(code int, body string) *httptest.Server { ...@@ -337,7 +286,7 @@ func testAuthServer(code int, body string) *httptest.Server {
func startServerOrFail(t *testing.T, ts *httptest.Server) *exec.Cmd { func startServerOrFail(t *testing.T, ts *httptest.Server) *exec.Cmd {
cmd := exec.Command("go", "run", "main.go", "upstream.go", "archive.go", "git-http.go", "helpers.go", "xsendfile.go", cmd := exec.Command("go", "run", "main.go", "upstream.go", "archive.go", "git-http.go", "helpers.go", "xsendfile.go",
"authorization.go", fmt.Sprintf("-authBackend=%s", ts.URL), fmt.Sprintf("-listenAddr=%s", servAddr)) "authorization.go", "lfs.go", fmt.Sprintf("-authBackend=%s", ts.URL), fmt.Sprintf("-listenAddr=%s", servAddr))
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
cmd.Stdout = os.Stdout cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr cmd.Stderr = os.Stderr
...@@ -401,3 +350,85 @@ func repoPath(t *testing.T) string { ...@@ -401,3 +350,85 @@ func repoPath(t *testing.T) string {
} }
return path.Join(cwd, testRepoRoot, testRepo) return path.Join(cwd, testRepoRoot, testRepo)
} }
func TestDeniedLfsDownload(t *testing.T) {
contentFilename := "b68143e6463773b1b6c6fd009a76c32aeec041faff32ba2ed42fd7f708a17f80"
url := fmt.Sprintf("http://%s/gitlab-lfs/objects/%s", servAddr, contentFilename)
prepareDownloadDir(t)
deniedXSendfileDownload(t, contentFilename, url)
}
func TestAllowedLfsDownload(t *testing.T) {
contentFilename := "b68143e6463773b1b6c6fd009a76c32aeec041faff32ba2ed42fd7f708a17f80"
url := fmt.Sprintf("http://%s/gitlab-lfs/objects/%s", servAddr, contentFilename)
prepareDownloadDir(t)
allowedXSendfileDownload(t, contentFilename, url)
}
func allowedXSendfileDownload(t *testing.T, contentFilename string, url string) {
contentPath := path.Join(cacheDir, contentFilename)
prepareDownloadDir(t)
// Prepare test server and backend
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if xSendfileType := r.Header.Get("X-Sendfile-Type"); xSendfileType != "X-Sendfile" {
t.Fatalf(`X-Sendfile-Type want "X-Sendfile" got %q`, xSendfileType)
}
w.Header().Set("X-Sendfile", contentPath)
w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, contentFilename))
w.Header().Set("Content-Type", fmt.Sprintf(`application/octet-stream`))
w.WriteHeader(200)
}))
defer ts.Close()
defer cleanUpProcessGroup(startServerOrFail(t, ts))
if err := os.MkdirAll(cacheDir, 0755); err != nil {
t.Fatal(err)
}
contentBytes := []byte{'c', 'o', 'n', 't', 'e', 'n', 't'}
if err := ioutil.WriteFile(contentPath, contentBytes, 0644); err != nil {
t.Fatal(err)
}
downloadCmd := exec.Command("curl", "-J", "-O", url)
downloadCmd.Dir = scratchDir
runOrFail(t, downloadCmd)
actual, err := ioutil.ReadFile(path.Join(scratchDir, contentFilename))
if err != nil {
t.Fatal(err)
}
if bytes.Compare(actual, contentBytes) != 0 {
t.Fatal("Unexpected file contents in download")
}
}
func deniedXSendfileDownload(t *testing.T, contentFilename string, url string) {
prepareDownloadDir(t)
// Prepare test server and backend
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if xSendfileType := r.Header.Get("X-Sendfile-Type"); xSendfileType != "X-Sendfile" {
t.Fatalf(`X-Sendfile-Type want "X-Sendfile" got %q`, xSendfileType)
}
w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, contentFilename))
w.WriteHeader(200)
fmt.Fprint(w, "Denied")
}))
defer ts.Close()
defer cleanUpProcessGroup(startServerOrFail(t, ts))
downloadCmd := exec.Command("curl", "-J", "-O", url)
downloadCmd.Dir = scratchDir
runOrFail(t, downloadCmd)
actual, err := ioutil.ReadFile(path.Join(scratchDir, contentFilename))
if err != nil {
t.Fatal(err)
}
if bytes.Compare(actual, []byte{'D', 'e', 'n', 'i', 'e', 'd'}) != 0 {
t.Fatal("Unexpected file contents in download")
}
}
...@@ -44,6 +44,13 @@ type authorizationResponse struct { ...@@ -44,6 +44,13 @@ type authorizationResponse struct {
// CommitId is used do prevent race conditions between the 'time of check' // CommitId is used do prevent race conditions between the 'time of check'
// in the GitLab Rails app and the 'time of use' in gitlab-workhorse. // in the GitLab Rails app and the 'time of use' in gitlab-workhorse.
CommitId string CommitId string
// StoreLFSPath is provided by the GitLab Rails application
// to mark where the tmp file should be placed
StoreLFSPath string
// LFS object id
LfsOid string
// LFS object size
LfsSize int64
} }
// A gitReqest is an *http.Request decorated with attributes returned by the // A gitReqest is an *http.Request decorated with attributes returned by the
...@@ -65,6 +72,8 @@ var gitServices = [...]gitService{ ...@@ -65,6 +72,8 @@ var gitServices = [...]gitService{
gitService{"GET", regexp.MustCompile(`/repository/archive.tar.gz\z`), repoPreAuthorizeHandler(handleGetArchive)}, gitService{"GET", regexp.MustCompile(`/repository/archive.tar.gz\z`), repoPreAuthorizeHandler(handleGetArchive)},
gitService{"GET", regexp.MustCompile(`/repository/archive.tar.bz2\z`), repoPreAuthorizeHandler(handleGetArchive)}, gitService{"GET", regexp.MustCompile(`/repository/archive.tar.bz2\z`), repoPreAuthorizeHandler(handleGetArchive)},
gitService{"GET", regexp.MustCompile(`/uploads/`), handleSendFile}, gitService{"GET", regexp.MustCompile(`/uploads/`), handleSendFile},
gitService{"PUT", regexp.MustCompile(`/gitlab-lfs/objects/([0-9a-f]{64})/([0-9]+)\z`), lfsAuthorizeHandler(handleStoreLfsObject)},
gitService{"GET", regexp.MustCompile(`/gitlab-lfs/objects/([0-9a-f]{64})\z`), handleSendFile},
} }
func newUpstream(authBackend string, authTransport http.RoundTripper) *upstream { func newUpstream(authBackend string, authTransport http.RoundTripper) *upstream {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment