Handle lfs upload with middleware and callback.

archive.go

@@ -5,6 +5,7 @@ In this file we handle 'git archive' downloads
 package main
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -13,9 +14,8 @@ import (
 	"os"
 	"os/exec"
 	"path"
-	"time"
 	"path/filepath"
-	"errors"
+	"time"
 )
 
 func handleGetArchive(w http.ResponseWriter, r *gitRequest) {

git-http.go

@@ -9,8 +9,8 @@ import (
 	"fmt"
 	"io"
 	"net/http"
-	"strings"
 	"path/filepath"
+	"strings"
 )
 
 func handleGetInfoRefs(w http.ResponseWriter, r *gitRequest) {

lfs.go

@@ -5,11 +5,11 @@ In this file we handle git lfs objects downloads and uploads
 package main
 
 import (
-	"compress/gzip"
 	"crypto/sha256"
 	"encoding/hex"
 	"errors"
 	"io"
+	"io/ioutil"
 	"log"
 	"net/http"
 	"os"
@@ -23,100 +23,102 @@ var (
 	errSizeMismatch = errors.New("Content size does not match")
 )
 
-func handleStoreLfsObject(w http.ResponseWriter, r *gitRequest, rpc string) {
-	var body io.ReadCloser
+func lfsAuthorizeHandler(handleFunc serviceHandleFunc) serviceHandleFunc {
+	return preAuthorizeHandler(func(w http.ResponseWriter, r *gitRequest) {
 
-	urlPath := r.URL.Path
-	regExp := regexp.MustCompile(`([0-9a-f]{64})/([0-9]+)`)
-	matches := regExp.FindStringSubmatch(urlPath)
+		if r.StoreLFSPath == "" {
+			fail500(w, "lfsAuthorizeHandler", errors.New("Don't know where to store object, no store path specified."))
+			return
+		}
 
-	if matches == nil {
-		log.Printf("Found no object info in path: %s", urlPath)
-		return
-	}
+		handleFunc(w, r)
+	}, "")
+}
 
-	oid := matches[1]
-	size := matches[2]
-	log.Printf("Found oid: %s and size: %s", oid, size)
-
-	storePath := filepath.Join(r.StoreLFSPath, transformKey(oid))
-
-	if _, err := os.Stat(storePath); os.IsNotExist(err) {
-		tmpPath := filepath.Join(r.StoreLFSPath, "tmp", oid)
-
-		if _, err := os.Stat(tmpPath); os.IsNotExist(err) {
-			// TODO try removing gzip, possibly not needed
-			// The client request body may have been gzipped.
-			if r.Header.Get("Content-Encoding") == "gzip" {
-				body, err = gzip.NewReader(r.Body)
-				if err != nil {
-					fail500(w, "Couldn't handle LFS upload request.", err)
-					return
-				}
-			} else {
-				body = r.Body
-			}
-			defer body.Close()
-
-			// TODO maybe set dir permissions to 700
-			dir := filepath.Dir(tmpPath)
-			if err := os.MkdirAll(dir, 0750); err != nil {
-				fail500(w, "Couldn't create directory for storing LFS objects.", err)
-				return
-			}
-
-			// TODO use go library for creating TMP files
-			file, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0640)
-			if err != nil {
-				fail500(w, "Couldn't open tmp file for writing.", err)
-				return
-			}
-			// defer os.Remove(tmpPath)
-			defer file.Close()
-
-			hash := sha256.New()
-			hw := io.MultiWriter(hash, file)
-
-			written, err := io.Copy(hw, body)
-			if err != nil {
-				fail500(w, "Failed to save received LFS object.", err)
-				return
-			}
-			file.Close()
-
-			sizeInt, err := strconv.ParseInt(size, 10, 64)
-			if err != nil {
-				fail500(w, "Couldn't read size: ", err)
-				return
-			}
-
-			if written != sizeInt {
-				fail500(w, "Inconsistent size: ", errSizeMismatch)
-				return
-			}
-
-			shaStr := hex.EncodeToString(hash.Sum(nil))
-			if shaStr != oid {
-				fail500(w, "Inconsistent size: ", errSizeMismatch)
-				return
-			}
+func handleStoreLfsObject(handleFunc serviceHandleFunc) serviceHandleFunc {
+	return func(w http.ResponseWriter, r *gitRequest) {
+
+		urlPath := r.URL.Path
+		regExp := regexp.MustCompile(`([0-9a-f]{64})/([0-9]+)`)
+		matches := regExp.FindStringSubmatch(urlPath)
+
+		if matches == nil {
+			log.Printf("Found no object info in path: %s", urlPath)
+			return
 		}
-	}
-	// if err := os.Rename(tmpPath, path); err != nil {
-	// 	fail500(w, "Failed to rename temporary LFS object.", err)
-	// 	return
-	// }
 
-	log.Printf("Received the LFS object from client, oid: %s", oid)
+		oid := matches[1]
+		size := matches[2]
+		log.Printf("Found oid: %s and size: %s", oid, size)
 
-	return
+		sha := sha256.New()
+		sha.Write([]byte(oid))
+		tmp_hash := hex.EncodeToString(sha.Sum(nil))
+		tmpPath := filepath.Join(r.StoreLFSPath, "tmp")
+
+		var body io.ReadCloser
+
+		body = r.Body
+		defer body.Close()
+
+		dir := filepath.Dir(tmpPath)
+		if err := os.MkdirAll(dir, 0700); err != nil {
+			fail500(w, "Couldn't create directory for storing LFS objects.", err)
+			return
+		}
+
+		file, err := ioutil.TempFile(tmpPath, tmp_hash)
+		if err != nil {
+			fail500(w, "Couldn't open tmp file for writing.", err)
+			return
+		}
+		defer os.Remove(tmpPath)
+		defer file.Close()
 
+		hash := sha256.New()
+		hw := io.MultiWriter(hash, file)
+
+		written, err := io.Copy(hw, body)
+		if err != nil {
+			fail500(w, "Failed to save received LFS object.", err)
+			return
+		}
+		file.Close()
+
+		sizeInt, err := strconv.ParseInt(size, 10, 64)
+		if err != nil {
+			fail500(w, "Couldn't read size: ", err)
+			return
+		}
+
+		if written != sizeInt {
+			fail500(w, "Inconsistent size: ", errSizeMismatch)
+			return
+		}
+
+		shaStr := hex.EncodeToString(hash.Sum(nil))
+		if shaStr != oid {
+			fail500(w, "Inconsistent size: ", errSizeMismatch)
+			return
+		}
+		r.Header.Set("X-GitLab-Lfs-Tmp", filepath.Base(file.Name()))
+
+		handleFunc(w, r)
+	}
 }
 
-func transformKey(key string) string {
-	if len(key) < 5 {
-		return key
+func lfsCallback(w http.ResponseWriter, r *gitRequest) {
+	authReq, err := r.u.newUpstreamRequest(r.Request, nil, "/authorize")
+	if err != nil {
+		fail500(w, "newUpstreamRequestlfsCallback", err)
+		return
 	}
 
-	return filepath.Join(key[0:2], key[2:4], key[4:len(key)])
+	authResponse, err := r.u.httpClient.Do(authReq)
+	if err != nil {
+		fail500(w, "doRequestlfsCallback", err)
+		return
+	}
+	defer authResponse.Body.Close()
+	return
 }

upstream.go

@@ -45,7 +45,8 @@ type authorizationResponse struct {
 	// in the GitLab Rails app and the 'time of use' in gitlab-workhorse.
 	CommitId string
 
-	// TODO: say something about this
+	// StoreLFSPath is provided by the GitLab Rails application
+	// to mark where the tmp file should be placed
 	StoreLFSPath string
 }
 
@@ -54,7 +55,7 @@ type authorizationResponse struct {
 type gitRequest struct {
 	*http.Request
 	authorizationResponse
-	u   *upstream
+	u *upstream
 }
 
 // Routing table
@@ -68,7 +69,7 @@ var gitServices = [...]gitService{
 	gitService{"GET", regexp.MustCompile(`/repository/archive.tar.gz\z`), repoPreAuthorizeHandler(handleGetArchive)},
 	gitService{"GET", regexp.MustCompile(`/repository/archive.tar.bz2\z`), repoPreAuthorizeHandler(handleGetArchive)},
 	gitService{"GET", regexp.MustCompile(`/uploads/`), handleSendFile},
-	gitService{"PUT", regexp.MustCompile(`/gitlab-lfs/objects/([0-9a-f]{64})/([0-9]+)\z`), repoPreAuthorizeHandler(handleStoreLfsObject)},
+	gitService{"PUT", regexp.MustCompile(`/gitlab-lfs/objects/([0-9a-f]{64})/([0-9]+)\z`), lfsAuthorizeHandler(handleStoreLfsObject(lfsCallback))},
 	gitService{"GET", regexp.MustCompile(`/gitlab-lfs/objects/([0-9a-f]{64})\z`), handleSendFile},
 }