Merge pull request #32 from abiosoft/master

Implementation of Git middleware

config/middleware.go

@@ -7,6 +7,7 @@ import (
 	"github.com/mholt/caddy/middleware/errors"
 	"github.com/mholt/caddy/middleware/extensions"
 	"github.com/mholt/caddy/middleware/fastcgi"
+	"github.com/mholt/caddy/middleware/git"
 	"github.com/mholt/caddy/middleware/gzip"
 	"github.com/mholt/caddy/middleware/headers"
 	"github.com/mholt/caddy/middleware/log"
@@ -48,6 +49,7 @@ func init() {
 	register("ext", extensions.New)
 	register("basicauth", basicauth.New)
 	register("proxy", proxy.New)
+	register("git", git.New)
 	register("fastcgi", fastcgi.New)
 	register("websocket", websockets.New)
 	register("markdown", markdown.New)

middleware/git/doc.go

+// Package git is the middleware that pull sites from git repo
+//
+// Caddyfile Syntax :
+//	git repo path {
+//		repo
+//		path
+//		branch
+//		key
+//		interval
+//	}
+//	repo 	- git repository
+// 		compulsory. Both ssh (e.g. git@github.com:user/project.git)
+// 		and https(e.g. https://github.com/user/project) are supported.
+//		Can be specified in either config block or top level
+//
+// 	path 	- directory to pull into, relative to site root
+//		optional. Defaults to site root.
+//		If set, must be a subdirectory to site root to be valid.
+//
+// 	branch 	- git branch or tag
+//		optional. Defaults to master
+//
+// 	key 	- path to private ssh key
+//		optional. Required for private repositories. e.g. /home/user/.ssh/id_rsa
+//
+// 	interval- interval between git pulls in seconds
+//		optional. Defaults to 3600 (1 Hour).
+//
+// Examples :
+//
+// public repo pulled into site root
+//	git github.com/user/myproject
+//
+// public repo pulled into <root>/mysite
+//	git https://github.com/user/myproject mysite
+//
+// private repo pulled into <root>/mysite with tag v1.0 and interval of 1 day
+//	git {
+//		repo 	git@github.com:user/myproject
+//		branch 	v1.0
+//		path	mysite
+//		key 	/home/user/.ssh/id_rsa
+//		interval 86400 # 1 day
+//	}
+//
+// Caddyfile with private git repo and php support via fastcgi.
+// path defaults to /var/www/html/myphpsite as specified in root config.
+//
+//	0.0.0.0:8080
+//
+//	git {
+//		repo 	git@github.com:user/myphpsite
+//		key 	/home/user/.ssh/id_rsa
+//		interval 86400 # 1 day
+//	}
+//
+//	fastcgi / 127.0.0.1:9000 php
+//
+//	root /var/www/html/myphpsite
+//
+// A pull is first attempted after initialization. Afterwards, a pull is attempted
+// after request to server and if time taken since last successful pull is higher than interval.
+//
+// After the first successful pull (should be during initialization except an error occurs),
+// subsequent pulls are done in background and do not impact request time.
+//
+// Note: private repositories are currently only supported and tested on Linux and OSX
+package git

middleware/git/git.go

+package git
+
+import (
+	"fmt"
+	"github.com/mholt/caddy/middleware"
+	"net/http"
+	"net/url"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Git represents a middleware instance that pulls git repository.
+type Git struct {
+	Next middleware.Handler
+	Repo *Repo
+}
+
+// ServeHTTP satisfies the middleware.Handler interface.
+func (g Git) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) {
+	if err := g.Repo.Pull(); err != nil {
+		return 500, err
+	}
+	return g.Next.ServeHTTP(w, r)
+}
+
+// New creates a new instance of git middleware.
+func New(c middleware.Controller) (middleware.Middleware, error) {
+	repo, err := parse(c)
+	if err != nil {
+		return nil, err
+	}
+	err = repo.Pull()
+	return func(next middleware.Handler) middleware.Handler {
+		return Git{Next: next, Repo: repo}
+	}, err
+}
+
+func parse(c middleware.Controller) (*Repo, error) {
+	repo := &Repo{Branch: "master", Interval: DefaultInterval, Path: c.Root()}
+
+	for c.Next() {
+		args := c.RemainingArgs()
+
+		switch len(args) {
+		case 2:
+			repo.Path = filepath.Join(c.Root(), args[1])
+			fallthrough
+		case 1:
+			repo.Url = args[0]
+		}
+
+		for c.NextBlock() {
+			switch c.Val() {
+			case "repo":
+				if !c.NextArg() {
+					return nil, c.ArgErr()
+				}
+				repo.Url = c.Val()
+			case "path":
+				if !c.NextArg() {
+					return nil, c.ArgErr()
+				}
+				repo.Path = filepath.Join(c.Root(), c.Val())
+			case "branch":
+				if !c.NextArg() {
+					return nil, c.ArgErr()
+				}
+				repo.Branch = c.Val()
+			case "key":
+				if !c.NextArg() {
+					return nil, c.ArgErr()
+				}
+				repo.KeyPath = c.Val()
+			case "interval":
+				if !c.NextArg() {
+					return nil, c.ArgErr()
+				}
+				t, _ := strconv.Atoi(c.Val())
+				if t > 0 {
+					repo.Interval = time.Duration(t) * time.Second
+				}
+			}
+		}
+	}
+
+	// if repo is not specified, return error
+	if repo.Url == "" {
+		return nil, c.ArgErr()
+	}
+
+	// if private key is not specified, convert repository url to https
+	// to avoid ssh authentication
+	// else validate git url
+	// Note: private key support not yet available on Windows
+	var err error
+	if repo.KeyPath == "" {
+		repo.Url, repo.Host, err = sanitizeHttp(repo.Url)
+	} else {
+		repo.Url, repo.Host, err = sanitizeGit(repo.Url)
+		// TODO add Windows support for private repos
+		if runtime.GOOS == "windows" {
+			return nil, fmt.Errorf("Private repository not yet supported on Windows")
+		}
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	// validate git availability in PATH
+	if err = initGit(); err != nil {
+		return nil, err
+	}
+
+	return repo, prepare(repo)
+}
+
+// sanitizeHttp cleans up repository url and converts to https format
+// if currently in ssh format.
+// Returns sanitized url, hostName (e.g. github.com, bitbucket.com)
+// and possible error
+func sanitizeHttp(repoUrl string) (string, string, error) {
+	url, err := url.Parse(repoUrl)
+	if err != nil {
+		return "", "", err
+	}
+
+	if url.Host == "" && strings.HasPrefix(url.Path, "git@") {
+		url.Path = url.Path[len("git@"):]
+		i := strings.Index(url.Path, ":")
+		if i < 0 {
+			return "", "", fmt.Errorf("Invalid git url %s", repoUrl)
+		}
+		url.Host = url.Path[:i]
+		url.Path = "/" + url.Path[i+1:]
+	}
+
+	repoUrl = "https://" + url.Host + url.Path
+	return repoUrl, url.Host, nil
+}
+
+// sanitizeGit cleans up repository url and validate ssh format.
+// Returns sanitized url, hostName (e.g. github.com, bitbucket.com)
+// and possible error
+func sanitizeGit(repoUrl string) (string, string, error) {
+	repoUrl = strings.TrimSpace(repoUrl)
+	if !strings.HasPrefix(repoUrl, "git@") || strings.Index(repoUrl, ":") < len("git@a:") {
+		return "", "", fmt.Errorf("Invalid git url %s", repoUrl)
+	}
+	hostUrl := repoUrl[len("git@"):]
+	i := strings.Index(hostUrl, ":")
+	host := hostUrl[:i]
+	return repoUrl, host, nil
+}

middleware/git/gitclient.go

+package git
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"strings"
+	"sync"
+	"time"
+)
+
+// DefaultInterval is the minimum interval to delay before
+// requesting another git pull
+const DefaultInterval time.Duration = time.Hour * 1
+
+// gitBinary holds the absolute path to git executable
+var gitBinary string
+
+// initMutex prevents parallel attempt to validate
+// git availability in PATH
+var initMutex sync.Mutex = sync.Mutex{}
+
+// Repo is the structure that holds required information
+// of a git repository.
+type Repo struct {
+	Url      string        // Repository URL
+	Path     string        // Directory to pull to
+	Host     string        // Git domain host e.g. github.com
+	Branch   string        // Git branch
+	KeyPath  string        // Path to private ssh key
+	Interval time.Duration // Interval between pulls
+	pulled   bool          // true if there is a successful pull
+	lastPull time.Time     // time of the last successful pull
+	sync.Mutex
+}
+
+// Pull requests a repository pull.
+// If it has been performed previously, it returns
+// and requests another pull in background.
+// Otherwise it waits until the pull is done.
+func (r *Repo) Pull() error {
+	// if site is not pulled, pull
+	if !r.pulled {
+		return pull(r)
+	}
+
+	// request pull in background
+	go pull(r)
+	return nil
+}
+
+// pull performs git clone, or git pull if repository exists
+func pull(r *Repo) error {
+	r.Lock()
+	defer r.Unlock()
+	// if it is less than interval since last pull, return
+	if time.Since(r.lastPull) <= r.Interval {
+		return nil
+	}
+
+	params := []string{"clone", "-b", r.Branch, r.Url, r.Path}
+	if r.pulled {
+		params = []string{"pull", "origin", r.Branch}
+	}
+
+	// if key is specified, pull using ssh key
+	if r.KeyPath != "" {
+		return pullWithKey(r, params)
+	}
+
+	cmd := exec.Command(gitBinary, params...)
+	cmd.Env = os.Environ()
+	cmd.Stdout = os.Stderr
+	cmd.Stderr = os.Stderr
+	if r.pulled {
+		cmd.Dir = r.Path
+	}
+
+	var err error
+	if err = cmd.Start(); err != nil {
+		return err
+	}
+
+	if err = cmd.Wait(); err == nil {
+		r.pulled = true
+		r.lastPull = time.Now()
+		log.Printf("%v pulled.\n", r.Url)
+	}
+
+	return err
+}
+
+// pullWithKey performs git clone or git pull if repository exists.
+// It is used for private repositories and requires an ssh key.
+// Note: currently only limited to Linux and OSX.
+func pullWithKey(r *Repo, params []string) error {
+	var gitSsh, script *os.File
+	// ensure temporary files deleted after usage
+	defer func() {
+		if gitSsh != nil {
+			os.Remove(gitSsh.Name())
+		}
+		if script != nil {
+			os.Remove(script.Name())
+		}
+	}()
+
+	var err error
+	// write git.sh script to temp file
+	gitSsh, err = writeScriptFile(gitWrapperScript(gitBinary))
+	if err != nil {
+		return err
+	}
+
+	// write git clone bash script to file
+	script, err = writeScriptFile(bashScript(gitSsh.Name(), r, params))
+	if err != nil {
+		return err
+	}
+
+	// execute the git clone bash script
+	cmd := exec.Command(script.Name())
+	cmd.Env = os.Environ()
+	cmd.Stdout = os.Stderr
+	cmd.Stderr = os.Stderr
+	if r.pulled {
+		cmd.Dir = r.Path
+	}
+
+	if err = cmd.Start(); err != nil {
+		return err
+	}
+
+	if err = cmd.Wait(); err == nil {
+		r.pulled = true
+		r.lastPull = time.Now()
+		log.Printf("%v pulled.\n", r.Url)
+	}
+	return err
+}
+
+// prepare prepares for a git pull
+// and validates the configured directory
+func prepare(r *Repo) error {
+	// check if directory exists or is empty
+	// if not, create directory
+	fs, err := ioutil.ReadDir(r.Path)
+	if err != nil || len(fs) == 0 {
+		return os.MkdirAll(r.Path, os.FileMode(0755))
+	}
+
+	// validate git repo
+	isGit := false
+	for _, f := range fs {
+		if f.IsDir() && f.Name() == ".git" {
+			isGit = true
+			break
+		}
+	}
+
+	if isGit {
+		// check if same repository
+		var repoUrl string
+		if repoUrl, err = getRepoUrl(r.Path); err == nil && repoUrl == r.Url {
+			r.pulled = true
+			return nil
+		}
+		if err != nil {
+			return fmt.Errorf("Cannot retrieve repo url for %v Error: %v", r.Path, err)
+		}
+		return fmt.Errorf("Another git repo '%v' exists at %v", repoUrl, r.Path)
+	}
+	return fmt.Errorf("Cannot git clone into %v, directory not empty.", r.Path)
+}
+
+// getRepoUrl retrieves remote origin url for the git repository at path
+func getRepoUrl(path string) (string, error) {
+	args := []string{"config", "--get", "remote.origin.url"}
+
+	_, err := os.Stat(path)
+	if err != nil {
+		return "", err
+	}
+
+	cmd := exec.Command(gitBinary, args...)
+	cmd.Dir = path
+	output, err := cmd.Output()
+	if err != nil {
+		return "", err
+	}
+
+	return strings.TrimSpace(string(output)), nil
+}
+
+// initGit validates git installation and locates the git executable
+// binary in PATH
+func initGit() error {
+	// prevent concurrent call
+	initMutex.Lock()
+	defer initMutex.Unlock()
+
+	// if validation has been done before and binary located in
+	// PATH, return.
+	if gitBinary != "" {
+		return nil
+	}
+
+	// locate git binary in path
+	var err error
+	gitBinary, err = exec.LookPath("git")
+	return err
+
+}
+
+// writeScriptFile writes content to a temporary file.
+// It changes the temporary file mode to executable and
+// closes it to prepare it for execution.
+func writeScriptFile(content []byte) (file *os.File, err error) {
+	if file, err = ioutil.TempFile("", "caddy"); err != nil {
+		return nil, err
+	}
+	if _, err = file.Write(content); err != nil {
+		return nil, err
+	}
+	if err = file.Chmod(os.FileMode(0755)); err != nil {
+		return nil, err
+	}
+	return file, file.Close()
+}
+
+// gitWrapperScript forms content for git.sh script
+var gitWrapperScript = func(gitBinary string) []byte {
+	return []byte(fmt.Sprintf(`#!/bin/bash
+
+# The MIT License (MIT)
+# Copyright (c) 2013 Alvin Abad
+
+if [ $# -eq 0 ]; then
+    echo "Git wrapper script that can specify an ssh-key file
+Usage:
+    git.sh -i ssh-key-file git-command
+    "
+    exit 1
+fi
+
+# remove temporary file on exit
+trap 'rm -f /tmp/.git_ssh.$$' 0
+
+if [ "$1" = "-i" ]; then
+    SSH_KEY=$2; shift; shift
+    echo "ssh -i $SSH_KEY \$@" > /tmp/.git_ssh.$$
+    chmod +x /tmp/.git_ssh.$$
+    export GIT_SSH=/tmp/.git_ssh.$$
+fi
+
+# in case the git command is repeated
+[ "$1" = "git" ] && shift
+
+# Run the git command
+%v "$@"
+
+`, gitBinary))
+}
+
+// bashScript forms content of bash script to clone or update a repo using ssh
+var bashScript = func(gitShPath string, repo *Repo, params []string) []byte {
+	return []byte(fmt.Sprintf(`#!/bin/bash
+
+mkdir -p ~/.ssh;
+touch ~/.ssh/known_hosts;
+ssh-keyscan -t rsa,dsa %v 2>&1 | sort -u - ~/.ssh/known_hosts > ~/.ssh/tmp_hosts;
+cat ~/.ssh/tmp_hosts >> ~/.ssh/known_hosts;
+%v -i %v %v;
+`, repo.Host, gitShPath, repo.KeyPath, strings.Join(params, " ")))
+}