core: Major refactor for graceful restarts; numerous fixes

Merged config and app packages into one called caddy. Abstracted away caddy startup functionality making it easier to embed Caddy in any Go application and use it as a library. Graceful restart (should) now ensure child starts properly. Now piping a gob bundle to child process so that the child can match up inherited listeners to server address. Much cleanup still to do.

app/app.go

-// Package app holds application-global state to make it accessible
-// by other packages in the application.
-//
-// This package differs from config in that the things in app aren't
-// really related to server configuration.
-package app
-
-import (
-	"errors"
-	"log"
-	"os"
-	"os/signal"
-	"path/filepath"
-	"runtime"
-	"strconv"
-	"strings"
-	"sync"
-	"syscall"
-
-	"github.com/mholt/caddy/server"
-)
-
-const (
-	// Name is the program name
-	Name = "Caddy"
-
-	// Version is the program version
-	Version = "0.7.6"
-)
-
-var (
-	// Servers is a list of all the currently-listening servers
-	Servers []*server.Server
-
-	// ServersMutex protects the Servers slice during changes
-	ServersMutex sync.Mutex
-
-	// Wg is used to wait for all servers to shut down
-	Wg sync.WaitGroup
-
-	// HTTP2 indicates whether HTTP2 is enabled or not
-	HTTP2 bool // TODO: temporary flag until http2 is standard
-
-	// Quiet mode hides non-error initialization output
-	Quiet bool
-)
-
-func init() {
-	go func() {
-		// Wait for signal
-		interrupt := make(chan os.Signal, 1)
-		signal.Notify(interrupt, os.Interrupt, os.Kill) // TODO: syscall.SIGTERM? Or that should not run callbacks...
-		<-interrupt
-
-		// Run shutdown callbacks
-		var exitCode int
-		ServersMutex.Lock()
-		errs := server.ShutdownCallbacks(Servers)
-		ServersMutex.Unlock()
-		if len(errs) > 0 {
-			for _, err := range errs {
-				log.Println(err)
-			}
-			exitCode = 1
-		}
-		os.Exit(exitCode)
-	}()
-}
-
-// Restart restarts the entire application; gracefully with zero
-// downtime if on a POSIX-compatible system, or forcefully if on
-// Windows but with imperceptibly-short downtime.
-//
-// The restarted application will use caddyfile as its input
-// configuration; it will not look elsewhere for the config
-// to use.
-func Restart(caddyfile []byte) error {
-	// TODO: This is POSIX-only right now; also, os.Args[0] is required!
-	// TODO: Pipe the Caddyfile to stdin of child!
-	// TODO: Before stopping this process, verify child started successfully (valid Caddyfile, etc)
-
-	// Tell the child that it's a restart
-	os.Setenv("CADDY_RESTART", "true")
-
-	// Pass along current environment and file descriptors to child.
-	// We pass along the file descriptors explicitly to ensure proper
-	// order, since losing the original order will break the child.
-	fds := []uintptr{os.Stdin.Fd(), os.Stdout.Fd(), os.Stderr.Fd()}
-
-	// Now add file descriptors of the sockets
-	ServersMutex.Lock()
-	for _, s := range Servers {
-		fds = append(fds, s.ListenerFd())
-	}
-	ServersMutex.Unlock()
-
-	// Fork the process with the current environment and file descriptors
-	execSpec := &syscall.ProcAttr{
-		Env:   os.Environ(),
-		Files: fds,
-	}
-	fork, err := syscall.ForkExec(os.Args[0], os.Args, execSpec)
-	if err != nil {
-		log.Println("FORK ERR:", err, fork)
-	}
-
-	// Child process is listening now; we can stop all our servers here.
-	ServersMutex.Lock()
-	for _, s := range Servers {
-		go s.Stop() // TODO: error checking/reporting
-	}
-	ServersMutex.Unlock()
-
-	return err
-}
-
-// SetCPU parses string cpu and sets GOMAXPROCS
-// according to its value. It accepts either
-// a number (e.g. 3) or a percent (e.g. 50%).
-func SetCPU(cpu string) error {
-	var numCPU int
-
-	availCPU := runtime.NumCPU()
-
-	if strings.HasSuffix(cpu, "%") {
-		// Percent
-		var percent float32
-		pctStr := cpu[:len(cpu)-1]
-		pctInt, err := strconv.Atoi(pctStr)
-		if err != nil || pctInt < 1 || pctInt > 100 {
-			return errors.New("invalid CPU value: percentage must be between 1-100")
-		}
-		percent = float32(pctInt) / 100
-		numCPU = int(float32(availCPU) * percent)
-	} else {
-		// Number
-		num, err := strconv.Atoi(cpu)
-		if err != nil || num < 1 {
-			return errors.New("invalid CPU value: provide a number or percent greater than 0")
-		}
-		numCPU = num
-	}
-
-	if numCPU > availCPU {
-		numCPU = availCPU
-	}
-
-	runtime.GOMAXPROCS(numCPU)
-	return nil
-}
-
-// DataFolder returns the path to the folder
-// where the application may store data. This
-// currently resolves to ~/.caddy
-func DataFolder() string {
-	return filepath.Join(userHomeDir(), ".caddy")
-}
-
-// userHomeDir returns the user's home directory according to
-// environment variables.
-//
-// Credit: http://stackoverflow.com/a/7922977/1048862
-func userHomeDir() string {
-	if runtime.GOOS == "windows" {
-		home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
-		if home == "" {
-			home = os.Getenv("USERPROFILE")
-		}
-		return home
-	}
-	return os.Getenv("HOME")
-}

caddy/assets/path.go

+package assets
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+)
+
+// Path returns the path to the folder
+// where the application may store data. This
+// currently resolves to ~/.caddy
+func Path() string {
+	return filepath.Join(userHomeDir(), ".caddy")
+}
+
+// userHomeDir returns the user's home directory according to
+// environment variables.
+//
+// Credit: http://stackoverflow.com/a/7922977/1048862
+func userHomeDir() string {
+	if runtime.GOOS == "windows" {
+		home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
+		if home == "" {
+			home = os.Getenv("USERPROFILE")
+		}
+		return home
+	}
+	return os.Getenv("HOME")
+}

config/config.go → caddy/config.go

-package config
+package caddy
 
 import (
 	"fmt"
@@ -7,19 +7,14 @@ import (
 	"net"
 	"sync"
 
-	"github.com/mholt/caddy/app"
-	"github.com/mholt/caddy/config/letsencrypt"
-	"github.com/mholt/caddy/config/parse"
-	"github.com/mholt/caddy/config/setup"
+	"github.com/mholt/caddy/caddy/letsencrypt"
+	"github.com/mholt/caddy/caddy/parse"
+	"github.com/mholt/caddy/caddy/setup"
 	"github.com/mholt/caddy/middleware"
 	"github.com/mholt/caddy/server"
 )
 
 const (
-	DefaultHost = "0.0.0.0"
-	DefaultPort = "2015"
-	DefaultRoot = "."
-
 	// DefaultConfigFile is the name of the configuration file that is loaded
 	// by default if no other file is specified.
 	DefaultConfigFile = "Caddyfile"
@@ -56,8 +51,8 @@ func Load(filename string, input io.Reader) (Group, error) {
 				Root:       Root,
 				Middleware: make(map[string][]middleware.Middleware),
 				ConfigFile: filename,
-				AppName:    app.Name,
-				AppVersion: app.Version,
+				AppName:    AppName,
+				AppVersion: AppVersion,
 			}
 
 			// It is crucial that directives are executed in the proper order.

config/config_test.go → caddy/config_test.go

-package config
+package caddy
 
 import (
 	"testing"

config/directives.go → caddy/directives.go

-package config
+package caddy
 
 import (
-	"github.com/mholt/caddy/config/parse"
-	"github.com/mholt/caddy/config/setup"
+	"github.com/mholt/caddy/caddy/parse"
+	"github.com/mholt/caddy/caddy/setup"
 	"github.com/mholt/caddy/middleware"
 )
 

config/letsencrypt/letsencrypt.go → caddy/letsencrypt/letsencrypt.go

@@ -18,6 +18,12 @@ import (
 	"github.com/xenolf/lego/acme"
 )
 
+// OnRenew is the function that will be used to restart
+// the application or the part of the application that uses
+// the certificates maintained by this package. When at least
+// one certificate is renewed, this function will be called.
+var OnRenew func() error
+
 // Activate sets up TLS for each server config in configs
 // as needed. It only skips the config if the cert and key
 // are already provided or if plaintext http is explicitly

config/letsencrypt/renew.go → caddy/letsencrypt/renew.go

@@ -17,10 +17,16 @@ import (
 func keepCertificatesRenewed(configs []server.Config) {
 	ticker := time.Tick(renewInterval)
 	for range ticker {
-		if errs := processCertificateRenewal(configs); len(errs) > 0 {
+		if n, errs := processCertificateRenewal(configs); len(errs) > 0 {
 			for _, err := range errs {
 				log.Printf("[ERROR] cert renewal: %v\n", err)
 			}
+			if n > 0 && OnRenew != nil {
+				err := OnRenew()
+				if err != nil {
+					log.Printf("[ERROR] onrenew callback: %v\n", err)
+				}
+			}
 		}
 	}
 }
@@ -28,9 +34,11 @@ func keepCertificatesRenewed(configs []server.Config) {
 // checkCertificateRenewal loops through all configured
 // sites and looks for certificates to renew. Nothing is mutated
 // through this function. The changes happen directly on disk.
-func processCertificateRenewal(configs []server.Config) []error {
-	var errs []error
+// It returns the number of certificates renewed and
+func processCertificateRenewal(configs []server.Config) (int, []error) {
 	log.Print("[INFO] Processing certificate renewals...")
+	var errs []error
+	var n int
 
 	for _, cfg := range configs {
 		// Host must be TLS-enabled and have assets managed by LE
@@ -95,11 +103,12 @@ func processCertificateRenewal(configs []server.Config) []error {
 			}
 
 			saveCertsAndKeys([]acme.CertificateResource{newCertMeta})
+			n++
 		} else if daysLeft <= 14 {
 			// Warn on 14 days remaining
 			log.Printf("[WARN] There are %d days left on the certificate for %s. Will renew when 7 days remain.\n", daysLeft, cfg.Host)
 		}
 	}
 
-	return errs
+	return n, errs
 }

config/letsencrypt/storage.go → caddy/letsencrypt/storage.go

@@ -4,13 +4,13 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/mholt/caddy/app"
+	"github.com/mholt/caddy/caddy/assets"
 )
 
 // storage is used to get file paths in a consistent,
 // cross-platform way for persisting Let's Encrypt assets
 // on the file system.
-var storage = Storage(filepath.Join(app.DataFolder(), "letsencrypt"))
+var storage = Storage(filepath.Join(assets.Path(), "letsencrypt"))
 
 // Storage is a root directory and facilitates
 // forming file paths derived from it.

config/setup/controller.go → caddy/setup/controller.go

@@ -5,7 +5,7 @@ import (
 	"net/http"
 	"strings"
 
-	"github.com/mholt/caddy/config/parse"
+	"github.com/mholt/caddy/caddy/parse"
 	"github.com/mholt/caddy/middleware"
 	"github.com/mholt/caddy/server"
 )

main.go

 package main
 
 import (
-	"bytes"
+	"errors"
 	"flag"
 	"fmt"
 	"io/ioutil"
 	"log"
-	"net"
 	"os"
-	"os/exec"
-	"path"
 	"runtime"
 	"strconv"
 	"strings"
 	"time"
 
-	"github.com/mholt/caddy/app"
-	"github.com/mholt/caddy/config"
-	"github.com/mholt/caddy/config/letsencrypt"
-	"github.com/mholt/caddy/server"
+	"github.com/mholt/caddy/caddy"
+	"github.com/mholt/caddy/caddy/letsencrypt"
 )
 
 var (
@@ -28,25 +23,33 @@ var (
 	revoke  string
 )
 
+const (
+	appName    = "Caddy"
+	appVersion = "0.8 beta"
+)
+
 func init() {
-	flag.StringVar(&conf, "conf", "", "Configuration file to use (default="+config.DefaultConfigFile+")")
-	flag.BoolVar(&app.HTTP2, "http2", true, "Enable HTTP/2 support") // TODO: temporary flag until http2 merged into std lib
-	flag.BoolVar(&app.Quiet, "quiet", false, "Quiet mode (no initialization output)")
+	flag.StringVar(&conf, "conf", "", "Configuration file to use (default="+caddy.DefaultConfigFile+")")
+	flag.BoolVar(&caddy.HTTP2, "http2", true, "Enable HTTP/2 support") // TODO: temporary flag until http2 merged into std lib
+	flag.BoolVar(&caddy.Quiet, "quiet", false, "Quiet mode (no initialization output)")
 	flag.StringVar(&cpu, "cpu", "100%", "CPU cap")
-	flag.StringVar(&config.Root, "root", config.DefaultRoot, "Root path to default site")
-	flag.StringVar(&config.Host, "host", config.DefaultHost, "Default host")
-	flag.StringVar(&config.Port, "port", config.DefaultPort, "Default port")
+	flag.StringVar(&caddy.Root, "root", caddy.DefaultRoot, "Root path to default site")
+	flag.StringVar(&caddy.Host, "host", caddy.DefaultHost, "Default host")
+	flag.StringVar(&caddy.Port, "port", caddy.DefaultPort, "Default port")
 	flag.BoolVar(&version, "version", false, "Show version")
 	flag.BoolVar(&letsencrypt.Agreed, "agree", false, "Agree to Let's Encrypt Subscriber Agreement")
 	flag.StringVar(&letsencrypt.DefaultEmail, "email", "", "Default email address to use for Let's Encrypt transactions")
-	flag.StringVar(&revoke, "revoke", "", "Hostname for which to revoke the certificate")
+	flag.StringVar(&revoke, "revoke", "", "Hostname for which to revoke its certificate")
 }
 
 func main() {
 	flag.Parse()
 
+	caddy.AppName = appName
+	caddy.AppVersion = appVersion
+
 	if version {
-		fmt.Printf("%s %s\n", app.Name, app.Version)
+		fmt.Printf("%s %s\n", caddy.AppName, caddy.AppVersion)
 		os.Exit(0)
 	}
 	if revoke != "" {
@@ -59,165 +62,103 @@ func main() {
 	}
 
 	// Set CPU cap
-	err := app.SetCPU(cpu)
+	err := setCPU(cpu)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	// Load config from file
-	groupings, err := loadConfigs()
+	// Get Caddyfile input
+	caddyfile, err := caddy.LoadCaddyfile(loadCaddyfile)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	// Start each server with its one or more configurations
-	for i, group := range groupings {
-		s, err := server.New(group.BindAddr.String(), group.Configs)
-		if err != nil {
-			log.Fatal(err)
-		}
-		s.HTTP2 = app.HTTP2 // TODO: This setting is temporary
-
-		app.Wg.Add(1)
-		go func(s *server.Server, i int) {
-			defer app.Wg.Done()
-
-			if os.Getenv("CADDY_RESTART") == "true" {
-				file := os.NewFile(uintptr(3+i), "")
-				ln, err := net.FileListener(file)
-				if err != nil {
-					log.Fatal("FILE LISTENER:", err)
-				}
-
-				lnf, ok := ln.(server.ListenerFile)
-				if !ok {
-					log.Fatal("Listener was not a ListenerFile")
-				}
-
-				err = s.Serve(lnf)
-				// TODO: Better error logging... also, is it even necessary?
-				if err != nil {
-					log.Println(err)
-				}
-			} else {
-				err := s.ListenAndServe()
-				// TODO: Better error logging... also, is it even necessary?
-				// For example, "use of closed network connection" is normal if doing graceful shutdown...
-				if err != nil {
-					log.Println(err)
-				}
-			}
-		}(s, i)
-
-		app.ServersMutex.Lock()
-		app.Servers = append(app.Servers, s)
-		app.ServersMutex.Unlock()
-	}
-
-	// Show initialization output
-	if !app.Quiet {
-		var checkedFdLimit bool
-		for _, group := range groupings {
-			for _, conf := range group.Configs {
-				// Print address of site
-				fmt.Println(conf.Address())
-
-				// Note if non-localhost site resolves to loopback interface
-				if group.BindAddr.IP.IsLoopback() && !isLocalhost(conf.Host) {
-					fmt.Printf("Notice: %s is only accessible on this machine (%s)\n",
-						conf.Host, group.BindAddr.IP.String())
-				}
-				if !checkedFdLimit && !group.BindAddr.IP.IsLoopback() && !isLocalhost(conf.Host) {
-					checkFdlimit()
-					checkedFdLimit = true
-				}
-			}
-		}
+	// Start your engines
+	err = caddy.Start(caddyfile)
+	if err != nil {
+		log.Fatal(err)
 	}
 
 	// TODO: Temporary; testing restart
-	if os.Getenv("CADDY_RESTART") != "true" {
-		go func() {
-			time.Sleep(5 * time.Second)
-			fmt.Println("restarting")
-			log.Println("RESTART ERR:", app.Restart([]byte{}))
-		}()
-	}
-
-	// Wait for all servers to be stopped
-	app.Wg.Wait()
-}
-
-// checkFdlimit issues a warning if the OS max file descriptors is below a recommended minimum.
-func checkFdlimit() {
-	const min = 4096
-
-	// Warn if ulimit is too low for production sites
-	if runtime.GOOS == "linux" || runtime.GOOS == "darwin" {
-		out, err := exec.Command("sh", "-c", "ulimit -n").Output() // use sh because ulimit isn't in Linux $PATH
-		if err == nil {
-			// Note that an error here need not be reported
-			lim, err := strconv.Atoi(string(bytes.TrimSpace(out)))
-			if err == nil && lim < min {
-				fmt.Printf("Warning: File descriptor limit %d is too low for production sites. At least %d is recommended. Set with \"ulimit -n %d\".\n", lim, min, min)
-			}
-		}
-	}
-}
-
-// isLocalhost returns true if the string looks explicitly like a localhost address.
-func isLocalhost(s string) bool {
-	return s == "localhost" || s == "::1" || strings.HasPrefix(s, "127.")
+	//if os.Getenv("CADDY_RESTART") != "true" {
+	go func() {
+		time.Sleep(5 * time.Second)
+		fmt.Println("restarting")
+		log.Println("RESTART ERR:", caddy.Restart(nil))
+	}()
+	//}
+
+	// Twiddle your thumbs
+	caddy.Wait()
 }
 
-// loadConfigs loads configuration from a file or stdin (piped).
-// The configurations are grouped by bind address.
-// Configuration is obtained from one of four sources, tried
-// in this order: 1. -conf flag, 2. stdin, 3. command line argument 4. Caddyfile.
-// If none of those are available, a default configuration is loaded.
-func loadConfigs() (config.Group, error) {
+func loadCaddyfile() (caddy.Input, error) {
 	// -conf flag
 	if conf != "" {
-		file, err := os.Open(conf)
+		contents, err := ioutil.ReadFile(conf)
 		if err != nil {
 			return nil, err
 		}
-		defer file.Close()
-		return config.Load(path.Base(conf), file)
-	}
-
-	// stdin
-	fi, err := os.Stdin.Stat()
-	if err == nil && fi.Mode()&os.ModeCharDevice == 0 {
-		// Note that a non-nil error is not a problem. Windows
-		// will not create a stdin if there is no pipe, which
-		// produces an error when calling Stat(). But Unix will
-		// make one either way, which is why we also check that
-		// bitmask.
-		confBody, err := ioutil.ReadAll(os.Stdin)
-		if err != nil {
-			log.Fatal(err)
-		}
-		if len(confBody) > 0 {
-			return config.Load("stdin", bytes.NewReader(confBody))
-		}
+		return caddy.CaddyfileInput{
+			Contents: contents,
+			Filepath: conf,
+		}, nil
 	}
 
-	// Command line args
+	// command line args
 	if flag.NArg() > 0 {
-		confBody := ":" + config.DefaultPort + "\n" + strings.Join(flag.Args(), "\n")
-		return config.Load("args", bytes.NewBufferString(confBody))
+		confBody := ":" + caddy.DefaultPort + "\n" + strings.Join(flag.Args(), "\n")
+		return caddy.CaddyfileInput{
+			Contents: []byte(confBody),
+			Filepath: "args",
+		}, nil
 	}
 
-	// Caddyfile
-	file, err := os.Open(config.DefaultConfigFile)
+	// Caddyfile in cwd
+	contents, err := ioutil.ReadFile(caddy.DefaultConfigFile)
 	if err != nil {
 		if os.IsNotExist(err) {
-			return config.Default()
+			return caddy.DefaultInput, nil
 		}
 		return nil, err
 	}
-	defer file.Close()
+	return caddy.CaddyfileInput{
+		Contents: contents,
+		Filepath: caddy.DefaultConfigFile,
+	}, nil
+}
+
+// setCPU parses string cpu and sets GOMAXPROCS
+// according to its value. It accepts either
+// a number (e.g. 3) or a percent (e.g. 50%).
+func setCPU(cpu string) error {
+	var numCPU int
+
+	availCPU := runtime.NumCPU()
+
+	if strings.HasSuffix(cpu, "%") {
+		// Percent
+		var percent float32
+		pctStr := cpu[:len(cpu)-1]
+		pctInt, err := strconv.Atoi(pctStr)
+		if err != nil || pctInt < 1 || pctInt > 100 {
+			return errors.New("invalid CPU value: percentage must be between 1-100")
+		}
+		percent = float32(pctInt) / 100
+		numCPU = int(float32(availCPU) * percent)
+	} else {
+		// Number
+		num, err := strconv.Atoi(cpu)
+		if err != nil || num < 1 {
+			return errors.New("invalid CPU value: provide a number or percent greater than 0")
+		}
+		numCPU = num
+	}
+
+	if numCPU > availCPU {
+		numCPU = availCPU
+	}
 
-	return config.Load(config.DefaultConfigFile, file)
+	runtime.GOMAXPROCS(numCPU)
+	return nil
 }

middleware/proxy/upstream.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/mholt/caddy/config/parse"
+	"github.com/mholt/caddy/caddy/parse"
 )
 
 var (

server/graceful.go

@@ -65,6 +65,12 @@ type gracefulConn struct {
 
 // Close closes c's underlying connection while updating the wg count.
 func (c gracefulConn) Close() error {
+	err := c.Conn.Close()
+	if err != nil {
+		return err
+	}
+	// close can fail on http2 connections (as of Oct. 2015, before http2 in std lib)
+	// so don't decrement count unless close succeeds
 	c.httpWg.Done()
-	return c.Conn.Close()
+	return nil
 }

server/server.go

@@ -59,14 +59,13 @@ func New(addr string, configs []Config) (*Server, error) {
 		tls:    tls,
 		vhosts: make(map[string]virtualHost),
 	}
-	s.Handler = s // TODO: this is weird
+	s.Handler = s // this is weird, but whatever
 
 	// We have to bound our wg with one increment
 	// to prevent a "race condition" that is hard-coded
 	// into sync.WaitGroup.Wait() - basically, an add
 	// with a positive delta must be guaranteed to
 	// occur before Wait() is called on the wg.
-	fmt.Println("+1 (new)")
 	s.httpWg.Add(1)
 
 	// Set up each virtualhost
@@ -169,11 +168,6 @@ func (s *Server) setup() error {
 // by the Go Authors. It has been modified to support multiple certificate/key pairs,
 // client authentication, and our custom Server type.
 func serveTLSWithSNI(s *Server, ln net.Listener, tlsConfigs []TLSConfig) error {
-	addr := s.Server.Addr
-	if addr == "" {
-		addr = ":https"
-	}
-
 	config := cloneTLSConfig(s.TLSConfig)
 	if config.NextProtos == nil {
 		config.NextProtos = []string{"http/1.1"}
@@ -267,7 +261,7 @@ func (s *Server) ListenerFd() uintptr {
 // (configuration and middleware stack) will handle the request.
 func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	fmt.Println("Sleeping")
-	time.Sleep(5 * time.Second)
+	time.Sleep(5 * time.Second) // TODO: Temporarily making requests hang so we can test graceful restart
 	fmt.Println("Unblocking")
 	defer func() {
 		// In case the user doesn't enable error middleware, we still