Commit e6c7d974 authored by Marin Jankovski's avatar Marin Jankovski

Handle lfs upload with middleware and callback.

parent ec721eef
...@@ -5,6 +5,7 @@ In this file we handle 'git archive' downloads ...@@ -5,6 +5,7 @@ In this file we handle 'git archive' downloads
package main package main
import ( import (
"errors"
"fmt" "fmt"
"io" "io"
"io/ioutil" "io/ioutil"
...@@ -13,9 +14,8 @@ import ( ...@@ -13,9 +14,8 @@ import (
"os" "os"
"os/exec" "os/exec"
"path" "path"
"time"
"path/filepath" "path/filepath"
"errors" "time"
) )
func handleGetArchive(w http.ResponseWriter, r *gitRequest) { func handleGetArchive(w http.ResponseWriter, r *gitRequest) {
......
...@@ -9,8 +9,8 @@ import ( ...@@ -9,8 +9,8 @@ import (
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
"strings"
"path/filepath" "path/filepath"
"strings"
) )
func handleGetInfoRefs(w http.ResponseWriter, r *gitRequest) { func handleGetInfoRefs(w http.ResponseWriter, r *gitRequest) {
......
...@@ -5,11 +5,11 @@ In this file we handle git lfs objects downloads and uploads ...@@ -5,11 +5,11 @@ In this file we handle git lfs objects downloads and uploads
package main package main
import ( import (
"compress/gzip"
"crypto/sha256" "crypto/sha256"
"encoding/hex" "encoding/hex"
"errors" "errors"
"io" "io"
"io/ioutil"
"log" "log"
"net/http" "net/http"
"os" "os"
...@@ -23,8 +23,20 @@ var ( ...@@ -23,8 +23,20 @@ var (
errSizeMismatch = errors.New("Content size does not match") errSizeMismatch = errors.New("Content size does not match")
) )
func handleStoreLfsObject(w http.ResponseWriter, r *gitRequest, rpc string) { func lfsAuthorizeHandler(handleFunc serviceHandleFunc) serviceHandleFunc {
var body io.ReadCloser return preAuthorizeHandler(func(w http.ResponseWriter, r *gitRequest) {
if r.StoreLFSPath == "" {
fail500(w, "lfsAuthorizeHandler", errors.New("Don't know where to store object, no store path specified."))
return
}
handleFunc(w, r)
}, "")
}
func handleStoreLfsObject(handleFunc serviceHandleFunc) serviceHandleFunc {
return func(w http.ResponseWriter, r *gitRequest) {
urlPath := r.URL.Path urlPath := r.URL.Path
regExp := regexp.MustCompile(`([0-9a-f]{64})/([0-9]+)`) regExp := regexp.MustCompile(`([0-9a-f]{64})/([0-9]+)`)
...@@ -39,39 +51,28 @@ func handleStoreLfsObject(w http.ResponseWriter, r *gitRequest, rpc string) { ...@@ -39,39 +51,28 @@ func handleStoreLfsObject(w http.ResponseWriter, r *gitRequest, rpc string) {
size := matches[2] size := matches[2]
log.Printf("Found oid: %s and size: %s", oid, size) log.Printf("Found oid: %s and size: %s", oid, size)
storePath := filepath.Join(r.StoreLFSPath, transformKey(oid)) sha := sha256.New()
sha.Write([]byte(oid))
tmp_hash := hex.EncodeToString(sha.Sum(nil))
tmpPath := filepath.Join(r.StoreLFSPath, "tmp")
if _, err := os.Stat(storePath); os.IsNotExist(err) { var body io.ReadCloser
tmpPath := filepath.Join(r.StoreLFSPath, "tmp", oid)
if _, err := os.Stat(tmpPath); os.IsNotExist(err) {
// TODO try removing gzip, possibly not needed
// The client request body may have been gzipped.
if r.Header.Get("Content-Encoding") == "gzip" {
body, err = gzip.NewReader(r.Body)
if err != nil {
fail500(w, "Couldn't handle LFS upload request.", err)
return
}
} else {
body = r.Body body = r.Body
}
defer body.Close() defer body.Close()
// TODO maybe set dir permissions to 700
dir := filepath.Dir(tmpPath) dir := filepath.Dir(tmpPath)
if err := os.MkdirAll(dir, 0750); err != nil { if err := os.MkdirAll(dir, 0700); err != nil {
fail500(w, "Couldn't create directory for storing LFS objects.", err) fail500(w, "Couldn't create directory for storing LFS objects.", err)
return return
} }
// TODO use go library for creating TMP files file, err := ioutil.TempFile(tmpPath, tmp_hash)
file, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0640)
if err != nil { if err != nil {
fail500(w, "Couldn't open tmp file for writing.", err) fail500(w, "Couldn't open tmp file for writing.", err)
return return
} }
// defer os.Remove(tmpPath) defer os.Remove(tmpPath)
defer file.Close() defer file.Close()
hash := sha256.New() hash := sha256.New()
...@@ -100,23 +101,24 @@ func handleStoreLfsObject(w http.ResponseWriter, r *gitRequest, rpc string) { ...@@ -100,23 +101,24 @@ func handleStoreLfsObject(w http.ResponseWriter, r *gitRequest, rpc string) {
fail500(w, "Inconsistent size: ", errSizeMismatch) fail500(w, "Inconsistent size: ", errSizeMismatch)
return return
} }
} r.Header.Set("X-GitLab-Lfs-Tmp", filepath.Base(file.Name()))
}
// if err := os.Rename(tmpPath, path); err != nil {
// fail500(w, "Failed to rename temporary LFS object.", err)
// return
// }
log.Printf("Received the LFS object from client, oid: %s", oid)
return
handleFunc(w, r)
}
} }
func transformKey(key string) string { func lfsCallback(w http.ResponseWriter, r *gitRequest) {
if len(key) < 5 { authReq, err := r.u.newUpstreamRequest(r.Request, nil, "/authorize")
return key if err != nil {
fail500(w, "newUpstreamRequestlfsCallback", err)
return
} }
return filepath.Join(key[0:2], key[2:4], key[4:len(key)]) authResponse, err := r.u.httpClient.Do(authReq)
if err != nil {
fail500(w, "doRequestlfsCallback", err)
return
}
defer authResponse.Body.Close()
return
} }
...@@ -45,7 +45,8 @@ type authorizationResponse struct { ...@@ -45,7 +45,8 @@ type authorizationResponse struct {
// in the GitLab Rails app and the 'time of use' in gitlab-workhorse. // in the GitLab Rails app and the 'time of use' in gitlab-workhorse.
CommitId string CommitId string
// TODO: say something about this // StoreLFSPath is provided by the GitLab Rails application
// to mark where the tmp file should be placed
StoreLFSPath string StoreLFSPath string
} }
...@@ -68,7 +69,7 @@ var gitServices = [...]gitService{ ...@@ -68,7 +69,7 @@ var gitServices = [...]gitService{
gitService{"GET", regexp.MustCompile(`/repository/archive.tar.gz\z`), repoPreAuthorizeHandler(handleGetArchive)}, gitService{"GET", regexp.MustCompile(`/repository/archive.tar.gz\z`), repoPreAuthorizeHandler(handleGetArchive)},
gitService{"GET", regexp.MustCompile(`/repository/archive.tar.bz2\z`), repoPreAuthorizeHandler(handleGetArchive)}, gitService{"GET", regexp.MustCompile(`/repository/archive.tar.bz2\z`), repoPreAuthorizeHandler(handleGetArchive)},
gitService{"GET", regexp.MustCompile(`/uploads/`), handleSendFile}, gitService{"GET", regexp.MustCompile(`/uploads/`), handleSendFile},
gitService{"PUT", regexp.MustCompile(`/gitlab-lfs/objects/([0-9a-f]{64})/([0-9]+)\z`), repoPreAuthorizeHandler(handleStoreLfsObject)}, gitService{"PUT", regexp.MustCompile(`/gitlab-lfs/objects/([0-9a-f]{64})/([0-9]+)\z`), lfsAuthorizeHandler(handleStoreLfsObject(lfsCallback))},
gitService{"GET", regexp.MustCompile(`/gitlab-lfs/objects/([0-9a-f]{64})\z`), handleSendFile}, gitService{"GET", regexp.MustCompile(`/gitlab-lfs/objects/([0-9a-f]{64})\z`), handleSendFile},
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment