Merge pull request #290 from mholt/le-graceful

Graceful restarts/reloads, refactoring

app/app.go

-// Package app holds application-global state to make it accessible
-// by other packages in the application.
-//
-// This package differs from config in that the things in app aren't
-// really related to server configuration.
-package app
-
-import (
-	"errors"
-	"os"
-	"path/filepath"
-	"runtime"
-	"strconv"
-	"strings"
-	"sync"
-
-	"github.com/mholt/caddy/server"
-)
-
-const (
-	// Name is the program name
-	Name = "Caddy"
-
-	// Version is the program version
-	Version = "0.7.6"
-)
-
-var (
-	// Servers is a list of all the currently-listening servers
-	Servers []*server.Server
-
-	// ServersMutex protects the Servers slice during changes
-	ServersMutex sync.Mutex
-
-	// Wg is used to wait for all servers to shut down
-	Wg sync.WaitGroup
-
-	// HTTP2 indicates whether HTTP2 is enabled or not
-	HTTP2 bool // TODO: temporary flag until http2 is standard
-
-	// Quiet mode hides non-error initialization output
-	Quiet bool
-)
-
-// SetCPU parses string cpu and sets GOMAXPROCS
-// according to its value. It accepts either
-// a number (e.g. 3) or a percent (e.g. 50%).
-func SetCPU(cpu string) error {
-	var numCPU int
-
-	availCPU := runtime.NumCPU()
-
-	if strings.HasSuffix(cpu, "%") {
-		// Percent
-		var percent float32
-		pctStr := cpu[:len(cpu)-1]
-		pctInt, err := strconv.Atoi(pctStr)
-		if err != nil || pctInt < 1 || pctInt > 100 {
-			return errors.New("invalid CPU value: percentage must be between 1-100")
-		}
-		percent = float32(pctInt) / 100
-		numCPU = int(float32(availCPU) * percent)
-	} else {
-		// Number
-		num, err := strconv.Atoi(cpu)
-		if err != nil || num < 1 {
-			return errors.New("invalid CPU value: provide a number or percent greater than 0")
-		}
-		numCPU = num
-	}
-
-	if numCPU > availCPU {
-		numCPU = availCPU
-	}
-
-	runtime.GOMAXPROCS(numCPU)
-	return nil
-}
-
-// DataFolder returns the path to the folder
-// where the application may store data. This
-// currently resolves to ~/.caddy
-func DataFolder() string {
-	return filepath.Join(userHomeDir(), ".caddy")
-}
-
-// userHomeDir returns the user's home directory according to
-// environment variables.
-//
-// Credit: http://stackoverflow.com/a/7922977/1048862
-func userHomeDir() string {
-	if runtime.GOOS == "windows" {
-		home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
-		if home == "" {
-			home = os.Getenv("USERPROFILE")
-		}
-		return home
-	}
-	return os.Getenv("HOME")
-}

caddy/assets/path.go

+package assets
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+)
+
+// Path returns the path to the folder
+// where the application may store data. This
+// currently resolves to ~/.caddy
+func Path() string {
+	return filepath.Join(userHomeDir(), ".caddy")
+}
+
+// userHomeDir returns the user's home directory according to
+// environment variables.
+//
+// Credit: http://stackoverflow.com/a/7922977/1048862
+func userHomeDir() string {
+	if runtime.GOOS == "windows" {
+		home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
+		if home == "" {
+			home = os.Getenv("USERPROFILE")
+		}
+		return home
+	}
+	return os.Getenv("HOME")
+}

caddy/assets/path_test.go

+package assets
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestPath(t *testing.T) {
+	if actual := Path(); !strings.HasSuffix(actual, ".caddy") {
+		t.Errorf("Expected path to be a .caddy folder, got: %v", actual)
+	}
+}

caddy/caddy.go

+// Package caddy implements the Caddy web server as a service.
+//
+// To use this package, follow a few simple steps:
+//
+//   1. Set the AppName and AppVersion variables.
+//   2. Call LoadCaddyfile() to get the Caddyfile (it
+//      might have been piped in as part of a restart).
+//      You should pass in your own Caddyfile loader.
+//   3. Call caddy.Start() to start Caddy, caddy.Stop()
+//      to stop it, or caddy.Restart() to restart it.
+//
+// You should use caddy.Wait() to wait for all Caddy servers
+// to quit before your process exits.
+package caddy
+
+import (
+	"bytes"
+	"encoding/gob"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net"
+	"os"
+	"path"
+	"strings"
+	"sync"
+
+	"github.com/mholt/caddy/server"
+)
+
+// Configurable application parameters
+var (
+	// The name and version of the application.
+	AppName, AppVersion string
+
+	// If true, initialization will not show any output.
+	Quiet bool
+
+	// DefaultInput is the default configuration to use when config input is empty or missing.
+	DefaultInput = CaddyfileInput{
+		Contents: []byte(fmt.Sprintf("%s:%s\nroot %s", DefaultHost, DefaultPort, DefaultRoot)),
+	}
+
+	// HTTP2 indicates whether HTTP2 is enabled or not
+	HTTP2 bool // TODO: temporary flag until http2 is standard
+)
+
+var (
+	// caddyfile is the input configuration text used for this process
+	caddyfile Input
+
+	// caddyfileMu protects caddyfile during changes
+	caddyfileMu sync.Mutex
+
+	// incompleteRestartErr occurs if this process is a fork
+	// of the parent but no Caddyfile was piped in
+	incompleteRestartErr = errors.New("cannot finish restart successfully")
+
+	// servers is a list of all the currently-listening servers
+	servers []*server.Server
+
+	// serversMu protects the servers slice during changes
+	serversMu sync.Mutex
+
+	// wg is used to wait for all servers to shut down
+	wg sync.WaitGroup
+
+	// loadedGob is used if this is a child process as part of
+	// a graceful restart; it is used to map listeners to their
+	// index in the list of inherited file descriptors. This
+	// variable is not safe for concurrent access.
+	loadedGob caddyfileGob
+)
+
+const (
+	DefaultHost = "0.0.0.0"
+	DefaultPort = "2015"
+	DefaultRoot = "."
+)
+
+// Start starts Caddy with the given Caddyfile. If cdyfile
+// is nil or the process is forked from a parent as part of
+// a graceful restart, Caddy will check to see if Caddyfile
+// was piped from stdin and use that.
+//
+// If this process is a fork and no Caddyfile was piped in,
+// an error will be returned. If this process is NOT a fork
+// and cdyfile is nil, a default configuration will be assumed.
+// In any case, an error is returned if Caddy could not be
+// started.
+func Start(cdyfile Input) error {
+	var err error
+
+	// Input must never be nil; try to load something
+	if cdyfile == nil {
+		cdyfile, err = LoadCaddyfile(nil)
+		if err != nil {
+			return err
+		}
+	}
+
+	caddyfileMu.Lock()
+	caddyfile = cdyfile
+	caddyfileMu.Unlock()
+
+	groupings, err := Load(path.Base(cdyfile.Path()), bytes.NewReader(cdyfile.Body()))
+	if err != nil {
+		return err
+	}
+
+	// Start each server with its one or more configurations
+	err = startServers(groupings)
+	if err != nil {
+		return err
+	}
+
+	// Close remaining file descriptors we may have inherited that we don't need
+	if isRestart() {
+		for _, fdIndex := range loadedGob.ListenerFds {
+			file := os.NewFile(fdIndex, "")
+			fln, err := net.FileListener(file)
+			if err == nil {
+				fln.Close()
+			}
+		}
+	}
+
+	// Show initialization output
+	if !Quiet && !isRestart() {
+		var checkedFdLimit bool
+		for _, group := range groupings {
+			for _, conf := range group.Configs {
+				// Print address of site
+				fmt.Println(conf.Address())
+
+				// Note if non-localhost site resolves to loopback interface
+				if group.BindAddr.IP.IsLoopback() && !isLocalhost(conf.Host) {
+					fmt.Printf("Notice: %s is only accessible on this machine (%s)\n",
+						conf.Host, group.BindAddr.IP.String())
+				}
+				if !checkedFdLimit && !group.BindAddr.IP.IsLoopback() && !isLocalhost(conf.Host) {
+					checkFdlimit()
+					checkedFdLimit = true
+				}
+			}
+		}
+	}
+
+	// Tell parent process that we got this
+	if isRestart() {
+		ppipe := os.NewFile(3, "") // parent is listening on pipe at index 3
+		ppipe.Write([]byte("success"))
+		ppipe.Close()
+	}
+
+	return nil
+}
+
+// startServers starts all the servers in groupings,
+// taking into account whether or not this process is
+// a child from a graceful restart or not.
+func startServers(groupings Group) error {
+	for i, group := range groupings {
+		s, err := server.New(group.BindAddr.String(), group.Configs)
+		if err != nil {
+			log.Fatal(err)
+		}
+		s.HTTP2 = HTTP2 // TODO: This setting is temporary
+
+		var ln server.ListenerFile
+		if isRestart() {
+			// Look up this server's listener in the map of inherited file descriptors;
+			// if we don't have one, we must make a new one.
+			if fdIndex, ok := loadedGob.ListenerFds[s.Addr]; ok {
+				file := os.NewFile(fdIndex, "")
+
+				fln, err := net.FileListener(file)
+				if err != nil {
+					log.Fatal(err)
+				}
+
+				ln, ok = fln.(server.ListenerFile)
+				if !ok {
+					log.Fatal("listener was not a ListenerFile")
+				}
+
+				delete(loadedGob.ListenerFds, s.Addr) // mark it as used
+			}
+		}
+
+		wg.Add(1)
+		go func(s *server.Server, i int, ln server.ListenerFile) {
+			defer wg.Done()
+			if ln != nil {
+				err = s.Serve(ln)
+			} else {
+				err = s.ListenAndServe()
+			}
+
+			// "use of closed network connection" is normal if doing graceful shutdown...
+			if err != nil && !strings.Contains(err.Error(), "use of closed network connection") {
+				if isRestart() {
+					log.Fatal(err)
+				} else {
+					log.Println(err)
+				}
+			}
+		}(s, i, ln)
+
+		serversMu.Lock()
+		servers = append(servers, s)
+		serversMu.Unlock()
+	}
+	return nil
+}
+
+// Stop stops all servers. It blocks until they are all stopped.
+func Stop() error {
+	serversMu.Lock()
+	for _, s := range servers {
+		s.Stop() // TODO: error checking/reporting?
+	}
+	serversMu.Unlock()
+	return nil
+}
+
+// Wait blocks until all servers are stopped.
+func Wait() {
+	wg.Wait()
+}
+
+// LoadCaddyfile loads a Caddyfile in a way that prioritizes
+// reading from stdin pipe; otherwise it calls loader to load
+// the Caddyfile. If loader does not return a Caddyfile, the
+// default one will be returned. Thus, if there are no other
+// errors, this function always returns at least the default
+// Caddyfile.
+func LoadCaddyfile(loader func() (Input, error)) (cdyfile Input, err error) {
+	// If we are a fork, finishing the restart is highest priority;
+	// piped input is required in this case.
+	if isRestart() {
+		err := gob.NewDecoder(os.Stdin).Decode(&loadedGob)
+		if err != nil {
+			return nil, err
+		}
+		cdyfile = CaddyfileInput{
+			Filepath: os.Stdin.Name(),
+			Contents: loadedGob.Caddyfile,
+		}
+	}
+
+	// Otherwise, we first try to get from stdin pipe
+	if cdyfile == nil {
+		cdyfile, err = CaddyfileFromPipe(os.Stdin)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// No piped input, so try the user's loader instead
+	if cdyfile == nil && loader != nil {
+		cdyfile, err = loader()
+	}
+
+	// Otherwise revert to default
+	if cdyfile == nil {
+		cdyfile = DefaultInput
+	}
+
+	return
+}
+
+// CaddyfileFromPipe loads the Caddyfile input from f if f is
+// not interactive input. f is assumed to be a pipe or stream,
+// such as os.Stdin. If f is not a pipe, no error is returned
+// but the Input value will be nil. An error is only returned
+// if there was an error reading the pipe, even if the length
+// of what was read is 0.
+func CaddyfileFromPipe(f *os.File) (Input, error) {
+	fi, err := f.Stat()
+	if err == nil && fi.Mode()&os.ModeCharDevice == 0 {
+		// Note that a non-nil error is not a problem. Windows
+		// will not create a stdin if there is no pipe, which
+		// produces an error when calling Stat(). But Unix will
+		// make one either way, which is why we also check that
+		// bitmask.
+		// BUG: Reading from stdin after this fails (e.g. for the let's encrypt email address) (OS X)
+		confBody, err := ioutil.ReadAll(f)
+		if err != nil {
+			return nil, err
+		}
+		return CaddyfileInput{
+			Contents: confBody,
+			Filepath: f.Name(),
+		}, nil
+	}
+
+	// not having input from the pipe is not itself an error,
+	// just means no input to return.
+	return nil, nil
+}
+
+// Caddyfile returns the current Caddyfile
+func Caddyfile() Input {
+	caddyfileMu.Lock()
+	defer caddyfileMu.Unlock()
+	return caddyfile
+}
+
+// Input represents a Caddyfile; its contents and file path
+// (which should include the file name at the end of the path).
+// If path does not apply (e.g. piped input) you may use
+// any understandable value. The path is mainly used for logging,
+// error messages, and debugging.
+type Input interface {
+	// Gets the Caddyfile contents
+	Body() []byte
+
+	// Gets the path to the origin file
+	Path() string
+
+	// IsFile returns true if the original input was a file on the file system
+	// that could be loaded again later if requested.
+	IsFile() bool
+}

config/config.go → caddy/config.go

-package config
+package caddy
 
 import (
 	"fmt"
@@ -7,19 +7,14 @@ import (
 	"net"
 	"sync"
 
-	"github.com/mholt/caddy/app"
-	"github.com/mholt/caddy/config/letsencrypt"
-	"github.com/mholt/caddy/config/parse"
-	"github.com/mholt/caddy/config/setup"
+	"github.com/mholt/caddy/caddy/letsencrypt"
+	"github.com/mholt/caddy/caddy/parse"
+	"github.com/mholt/caddy/caddy/setup"
 	"github.com/mholt/caddy/middleware"
 	"github.com/mholt/caddy/server"
 )
 
 const (
-	DefaultHost = "0.0.0.0"
-	DefaultPort = "2015"
-	DefaultRoot = "."
-
 	// DefaultConfigFile is the name of the configuration file that is loaded
 	// by default if no other file is specified.
 	DefaultConfigFile = "Caddyfile"
@@ -56,8 +51,8 @@ func Load(filename string, input io.Reader) (Group, error) {
 				Root:       Root,
 				Middleware: make(map[string][]middleware.Middleware),
 				ConfigFile: filename,
-				AppName:    app.Name,
-				AppVersion: app.Version,
+				AppName:    AppName,
+				AppVersion: AppVersion,
 			}
 
 			// It is crucial that directives are executed in the proper order.
@@ -153,14 +148,14 @@ func makeStorages() map[string]interface{} {
 // bind address to list of configs that would become VirtualHosts on that
 // server. Use the keys of the returned map to create listeners, and use
 // the associated values to set up the virtualhosts.
-func arrangeBindings(allConfigs []server.Config) (map[*net.TCPAddr][]server.Config, error) {
-	addresses := make(map[*net.TCPAddr][]server.Config)
+func arrangeBindings(allConfigs []server.Config) (Group, error) {
+	var groupings Group
 
 	// Group configs by bind address
 	for _, conf := range allConfigs {
-		newAddr, warnErr, fatalErr := resolveAddr(conf)
+		bindAddr, warnErr, fatalErr := resolveAddr(conf)
 		if fatalErr != nil {
-			return addresses, fatalErr
+			return groupings, fatalErr
 		}
 		if warnErr != nil {
 			log.Println("[Warning]", warnErr)
@@ -169,37 +164,40 @@ func arrangeBindings(allConfigs []server.Config) (map[*net.TCPAddr][]server.Conf
 		// Make sure to compare the string representation of the address,
 		// not the pointer, since a new *TCPAddr is created each time.
 		var existing bool
-		for addr := range addresses {
-			if addr.String() == newAddr.String() {
-				addresses[addr] = append(addresses[addr], conf)
+		for i := 0; i < len(groupings); i++ {
+			if groupings[i].BindAddr.String() == bindAddr.String() {
+				groupings[i].Configs = append(groupings[i].Configs, conf)
 				existing = true
 				break
 			}
 		}
 		if !existing {
-			addresses[newAddr] = append(addresses[newAddr], conf)
+			groupings = append(groupings, BindingMapping{
+				BindAddr: bindAddr,
+				Configs:  []server.Config{conf},
+			})
 		}
 	}
 
 	// Don't allow HTTP and HTTPS to be served on the same address
-	for _, configs := range addresses {
-		isTLS := configs[0].TLS.Enabled
-		for _, config := range configs {
+	for _, group := range groupings {
+		isTLS := group.Configs[0].TLS.Enabled
+		for _, config := range group.Configs {
 			if config.TLS.Enabled != isTLS {
 				thisConfigProto, otherConfigProto := "HTTP", "HTTP"
 				if config.TLS.Enabled {
 					thisConfigProto = "HTTPS"
 				}
-				if configs[0].TLS.Enabled {
+				if group.Configs[0].TLS.Enabled {
 					otherConfigProto = "HTTPS"
 				}
-				return addresses, fmt.Errorf("configuration error: Cannot multiplex %s (%s) and %s (%s) on same address",
-					configs[0].Address(), otherConfigProto, config.Address(), thisConfigProto)
+				return groupings, fmt.Errorf("configuration error: Cannot multiplex %s (%s) and %s (%s) on same address",
+					group.Configs[0].Address(), otherConfigProto, config.Address(), thisConfigProto)
 			}
 		}
 	}
 
-	return addresses, nil
+	return groupings, nil
 }
 
 // resolveAddr determines the address (host and port) that a config will
@@ -291,5 +289,15 @@ var (
 	Port = DefaultPort
 )
 
+// BindingMapping maps a network address to configurations
+// that will bind to it. The order of the configs is important.
+type BindingMapping struct {
+	BindAddr *net.TCPAddr
+	Configs  []server.Config
+}
+
 // Group maps network addresses to their configurations.
-type Group map[*net.TCPAddr][]server.Config
+// Preserving the order of the groupings is important
+// (related to graceful shutdown and restart)
+// so this is a slice, not a literal map.
+type Group []BindingMapping

config/config_test.go → caddy/config_test.go

-package config
+package caddy
 
 import (
 	"testing"

config/directives.go → caddy/directives.go

-package config
+package caddy
 
 import (
-	"github.com/mholt/caddy/config/parse"
-	"github.com/mholt/caddy/config/setup"
+	"github.com/mholt/caddy/caddy/parse"
+	"github.com/mholt/caddy/caddy/setup"
 	"github.com/mholt/caddy/middleware"
 )
 

caddy/helpers.go

+package caddy
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"os/exec"
+	"runtime"
+	"strconv"
+	"strings"
+
+	"github.com/mholt/caddy/caddy/letsencrypt"
+)
+
+func init() {
+	letsencrypt.OnRenew = func() error { return Restart(nil) }
+}
+
+// isLocalhost returns true if the string looks explicitly like a localhost address.
+func isLocalhost(s string) bool {
+	return s == "localhost" || s == "::1" || strings.HasPrefix(s, "127.")
+}
+
+// checkFdlimit issues a warning if the OS max file descriptors is below a recommended minimum.
+func checkFdlimit() {
+	const min = 4096
+
+	// Warn if ulimit is too low for production sites
+	if runtime.GOOS == "linux" || runtime.GOOS == "darwin" {
+		out, err := exec.Command("sh", "-c", "ulimit -n").Output() // use sh because ulimit isn't in Linux $PATH
+		if err == nil {
+			// Note that an error here need not be reported
+			lim, err := strconv.Atoi(string(bytes.TrimSpace(out)))
+			if err == nil && lim < min {
+				fmt.Printf("Warning: File descriptor limit %d is too low for production sites. At least %d is recommended. Set with \"ulimit -n %d\".\n", lim, min, min)
+			}
+		}
+	}
+}
+
+// caddyfileGob maps bind address to index of the file descriptor
+// in the Files array passed to the child process. It also contains
+// the caddyfile contents. Used only during graceful restarts.
+type caddyfileGob struct {
+	ListenerFds map[string]uintptr
+	Caddyfile   []byte
+}
+
+// isRestart returns whether this process is, according
+// to env variables, a fork as part of a graceful restart.
+func isRestart() bool {
+	return os.Getenv("CADDY_RESTART") == "true"
+}
+
+// CaddyfileInput represents a Caddyfile as input
+// and is simply a convenient way to implement
+// the Input interface.
+type CaddyfileInput struct {
+	Filepath string
+	Contents []byte
+	RealFile bool
+}
+
+// Body returns c.Contents.
+func (c CaddyfileInput) Body() []byte { return c.Contents }
+
+// Path returns c.Filepath.
+func (c CaddyfileInput) Path() string { return c.Filepath }
+
+// Path returns true if the original input was a real file on the file system.
+func (c CaddyfileInput) IsFile() bool { return c.RealFile }

config/letsencrypt/crypto_test.go → caddy/letsencrypt/crypto_test.go

@@ -10,14 +10,14 @@ import (
 )
 
 func init() {
-	rsaKeySizeToUse = 128 // makes tests faster
+	rsaKeySizeToUse = 128 // make tests faster; small key size OK for testing
 }
 
 func TestSaveAndLoadRSAPrivateKey(t *testing.T) {
 	keyFile := "test.key"
 	defer os.Remove(keyFile)
 
-	privateKey, err := rsa.GenerateKey(rand.Reader, 128) // small key size is OK for testing
+	privateKey, err := rsa.GenerateKey(rand.Reader, rsaKeySizeToUse)
 	if err != nil {
 		t.Fatal(err)
 	}

config/letsencrypt/letsencrypt.go → caddy/letsencrypt/letsencrypt.go

@@ -18,6 +18,12 @@ import (
 	"github.com/xenolf/lego/acme"
 )
 
+// OnRenew is the function that will be used to restart
+// the application or the part of the application that uses
+// the certificates maintained by this package. When at least
+// one certificate is renewed, this function will be called.
+var OnRenew func() error
+
 // Activate sets up TLS for each server config in configs
 // as needed. It only skips the config if the cert and key
 // are already provided or if plaintext http is explicitly
@@ -33,7 +39,7 @@ import (
 func Activate(configs []server.Config) ([]server.Config, error) {
 	// First identify and configure any elligible hosts for which
 	// we already have certs and keys in storage from last time.
-	configLen := len(configs) // avoid infinite loop since this loop appends to the slice
+	configLen := len(configs) // avoid infinite loop since this loop appends plaintext to the slice
 	for i := 0; i < configLen; i++ {
 		if existingCertAndKey(configs[i].Host) && configs[i].TLS.LetsEncryptEmail != "off" {
 			configs = autoConfigure(&configs[i], configs)
@@ -152,7 +158,7 @@ func newClient(leEmail string) (*acme.Client, error) {
 	}
 
 	// The client facilitates our communication with the CA server.
-	client := acme.NewClient(caURL, &leUser, rsaKeySizeToUse, exposePort)
+	client := acme.NewClient(CAUrl, &leUser, rsaKeySizeToUse, exposePort)
 
 	// If not registered, the user must register an account with the CA
 	// and agree to terms
@@ -232,9 +238,14 @@ func saveCertsAndKeys(certificates []acme.CertificateResource) error {
 // autoConfigure enables TLS on cfg and appends, if necessary, a new config
 // to allConfigs that redirects plaintext HTTP to its new HTTPS counterpart.
 func autoConfigure(cfg *server.Config, allConfigs []server.Config) []server.Config {
-	bundleBytes, _ := ioutil.ReadFile(storage.SiteCertFile(cfg.Host))
-	ocsp, _ := acme.GetOCSPForCert(bundleBytes)
-	cfg.TLS.OCSPStaple = ocsp
+	bundleBytes, err := ioutil.ReadFile(storage.SiteCertFile(cfg.Host))
+	// TODO: Handle these errors better
+	if err == nil {
+		ocsp, err := acme.GetOCSPForCert(bundleBytes)
+		if err == nil {
+			cfg.TLS.OCSPStaple = ocsp
+		}
+	}
 	cfg.TLS.Certificate = storage.SiteCertFile(cfg.Host)
 	cfg.TLS.Key = storage.SiteKeyFile(cfg.Host)
 	cfg.TLS.Enabled = true
@@ -328,15 +339,13 @@ var (
 
 	// Whether user has agreed to the Let's Encrypt SA
 	Agreed bool
+
+	// The base URL to the CA's ACME endpoint
+	CAUrl string
 )
 
 // Some essential values related to the Let's Encrypt process
 const (
-	// The base URL to the Let's Encrypt CA
-	// TODO: Staging API URL is: https://acme-staging.api.letsencrypt.org
-	// TODO: Production endpoint is: https://acme-v01.api.letsencrypt.org
-	caURL = "http://192.168.99.100:4000"
-
 	// The port to expose to the CA server for Simple HTTP Challenge
 	exposePort = "5001"
 

config/letsencrypt/renew.go → caddy/letsencrypt/renew.go

@@ -17,10 +17,16 @@ import (
 func keepCertificatesRenewed(configs []server.Config) {
 	ticker := time.Tick(renewInterval)
 	for range ticker {
-		if errs := processCertificateRenewal(configs); len(errs) > 0 {
+		if n, errs := processCertificateRenewal(configs); len(errs) > 0 {
 			for _, err := range errs {
 				log.Printf("[ERROR] cert renewal: %v\n", err)
 			}
+			if n > 0 && OnRenew != nil {
+				err := OnRenew()
+				if err != nil {
+					log.Printf("[ERROR] onrenew callback: %v\n", err)
+				}
+			}
 		}
 	}
 }
@@ -28,9 +34,11 @@ func keepCertificatesRenewed(configs []server.Config) {
 // checkCertificateRenewal loops through all configured
 // sites and looks for certificates to renew. Nothing is mutated
 // through this function. The changes happen directly on disk.
-func processCertificateRenewal(configs []server.Config) []error {
-	var errs []error
+// It returns the number of certificates renewed and
+func processCertificateRenewal(configs []server.Config) (int, []error) {
 	log.Print("[INFO] Processing certificate renewals...")
+	var errs []error
+	var n int
 
 	for _, cfg := range configs {
 		// Host must be TLS-enabled and have assets managed by LE
@@ -96,11 +104,12 @@ func processCertificateRenewal(configs []server.Config) []error {
 			}
 
 			saveCertsAndKeys([]acme.CertificateResource{newCertMeta})
+			n++
 		} else if daysLeft <= 14 {
 			// Warn on 14 days remaining
 			log.Printf("[WARN] There are %d days left on the certificate for %s. Will renew when 7 days remain.\n", daysLeft, cfg.Host)
 		}
 	}
 
-	return errs
+	return n, errs
 }

config/letsencrypt/storage.go → caddy/letsencrypt/storage.go

@@ -4,13 +4,13 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/mholt/caddy/app"
+	"github.com/mholt/caddy/caddy/assets"
 )
 
 // storage is used to get file paths in a consistent,
 // cross-platform way for persisting Let's Encrypt assets
 // on the file system.
-var storage = Storage(filepath.Join(app.DataFolder(), "letsencrypt"))
+var storage = Storage(filepath.Join(assets.Path(), "letsencrypt"))
 
 // Storage is a root directory and facilitates
 // forming file paths derived from it.

caddy/restart.go

+// +build !windows
+
+package caddy
+
+import (
+	"encoding/gob"
+	"io/ioutil"
+	"log"
+	"os"
+	"syscall"
+)
+
+// Restart restarts the entire application; gracefully with zero
+// downtime if on a POSIX-compatible system, or forcefully if on
+// Windows but with imperceptibly-short downtime.
+//
+// The restarted application will use newCaddyfile as its input
+// configuration. If newCaddyfile is nil, the current (existing)
+// Caddyfile configuration will be used.
+func Restart(newCaddyfile Input) error {
+	if newCaddyfile == nil {
+		caddyfileMu.Lock()
+		newCaddyfile = caddyfile
+		caddyfileMu.Unlock()
+	}
+
+	if len(os.Args) == 0 { // this should never happen, but just in case...
+		os.Args = []string{""}
+	}
+
+	// Tell the child that it's a restart
+	os.Setenv("CADDY_RESTART", "true")
+
+	// Prepare our payload to the child process
+	cdyfileGob := caddyfileGob{
+		ListenerFds: make(map[string]uintptr),
+		Caddyfile:   newCaddyfile.Body(),
+	}
+
+	// Prepare a pipe to the fork's stdin so it can get the Caddyfile
+	rpipe, wpipe, err := os.Pipe()
+	if err != nil {
+		return err
+	}
+
+	// Prepare a pipe that the child process will use to communicate
+	// its success or failure with us, the parent
+	sigrpipe, sigwpipe, err := os.Pipe()
+	if err != nil {
+		return err
+	}
+
+	// Pass along current environment and file descriptors to child.
+	// Ordering here is very important: stdin, stdout, stderr, sigpipe,
+	// and then the listener file descriptors (in order).
+	fds := []uintptr{rpipe.Fd(), os.Stdout.Fd(), os.Stderr.Fd(), sigwpipe.Fd()}
+
+	// Now add file descriptors of the sockets
+	serversMu.Lock()
+	for i, s := range servers {
+		fds = append(fds, s.ListenerFd())
+		cdyfileGob.ListenerFds[s.Addr] = uintptr(4 + i) // 4 fds come before any of the listeners
+	}
+	serversMu.Unlock()
+
+	// Fork the process with the current environment and file descriptors
+	execSpec := &syscall.ProcAttr{
+		Env:   os.Environ(),
+		Files: fds,
+	}
+	_, err = syscall.ForkExec(os.Args[0], os.Args, execSpec)
+	if err != nil {
+		return err
+	}
+
+	// Feed it the Caddyfile
+	err = gob.NewEncoder(wpipe).Encode(cdyfileGob)
+	if err != nil {
+		return err
+	}
+	wpipe.Close()
+
+	// Wait for child process to signal success or fail
+	sigwpipe.Close() // close our copy of the write end of the pipe
+	answer, err := ioutil.ReadAll(sigrpipe)
+	if err != nil || len(answer) == 0 {
+		log.Println("restart: child failed to answer; changes not applied")
+		return incompleteRestartErr
+	}
+
+	// Child process is listening now; we can stop all our servers here.
+	return Stop()
+}

caddy/restart_windows.go

+package caddy
+
+func Restart(newCaddyfile Input) error {
+	if newCaddyfile == nil {
+		caddyfileMu.Lock()
+		newCaddyfile = caddyfile
+		caddyfileMu.Unlock()
+	}
+
+	wg.Add(1) // barrier so Wait() doesn't unblock
+
+	err := Stop()
+	if err != nil {
+		return err
+	}
+
+	err = Start(newCaddyfile)
+	if err != nil {
+		return err
+	}
+
+	wg.Done() // take down our barrier
+
+	return nil
+}

config/setup/controller.go → caddy/setup/controller.go

@@ -5,7 +5,7 @@ import (
 	"net/http"
 	"strings"
 
-	"github.com/mholt/caddy/config/parse"
+	"github.com/mholt/caddy/caddy/parse"
 	"github.com/mholt/caddy/middleware"
 	"github.com/mholt/caddy/server"
 )

caddy/sigtrap.go

+package caddy
+
+import (
+	"log"
+	"os"
+	"os/signal"
+
+	"github.com/mholt/caddy/server"
+)
+
+func init() {
+	// Trap quit signals (cross-platform)
+	go func() {
+		shutdown := make(chan os.Signal, 1)
+		signal.Notify(shutdown, os.Interrupt, os.Kill)
+		<-shutdown
+
+		var exitCode int
+
+		serversMu.Lock()
+		errs := server.ShutdownCallbacks(servers)
+		serversMu.Unlock()
+
+		if len(errs) > 0 {
+			for _, err := range errs {
+				log.Println(err)
+			}
+			exitCode = 1
+		}
+
+		os.Exit(exitCode)
+	}()
+}

caddy/sigtrap_posix.go

+// +build !windows
+
+package caddy
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+	"os/signal"
+	"syscall"
+)
+
+func init() {
+	// Trap POSIX-only signals
+	go func() {
+		reload := make(chan os.Signal, 1)
+		signal.Notify(reload, syscall.SIGUSR1) // reload configuration
+
+		for {
+			<-reload
+
+			var updatedCaddyfile Input
+
+			caddyfileMu.Lock()
+			if caddyfile.IsFile() {
+				body, err := ioutil.ReadFile(caddyfile.Path())
+				if err == nil {
+					caddyfile = CaddyfileInput{
+						Filepath: caddyfile.Path(),
+						Contents: body,
+						RealFile: true,
+					}
+				}
+			}
+			caddyfileMu.Unlock()
+
+			err := Restart(updatedCaddyfile)
+			if err != nil {
+				log.Println(err)
+			}
+		}
+	}()
+}

main.go

 package main
 
 import (
-	"bytes"
+	"errors"
 	"flag"
 	"fmt"
 	"io/ioutil"
 	"log"
 	"os"
-	"os/exec"
-	"path"
 	"runtime"
 	"strconv"
 	"strings"
 
-	"github.com/mholt/caddy/app"
-	"github.com/mholt/caddy/config"
-	"github.com/mholt/caddy/config/letsencrypt"
-	"github.com/mholt/caddy/server"
+	"github.com/mholt/caddy/caddy"
+	"github.com/mholt/caddy/caddy/letsencrypt"
 )
 
 var (
@@ -26,15 +22,24 @@ var (
 	revoke  string
 )
 
+const (
+	appName    = "Caddy"
+	appVersion = "0.8 beta"
+)
+
 func init() {
-	flag.StringVar(&conf, "conf", "", "Configuration file to use (default="+config.DefaultConfigFile+")")
-	flag.BoolVar(&app.HTTP2, "http2", true, "Enable HTTP/2 support") // TODO: temporary flag until http2 merged into std lib
-	flag.BoolVar(&app.Quiet, "quiet", false, "Quiet mode (no initialization output)")
+	flag.StringVar(&conf, "conf", "", "Configuration file to use (default="+caddy.DefaultConfigFile+")")
+	flag.BoolVar(&caddy.HTTP2, "http2", true, "HTTP/2 support") // TODO: temporary flag until http2 merged into std lib
+	flag.BoolVar(&caddy.Quiet, "quiet", false, "Quiet mode (no initialization output)")
 	flag.StringVar(&cpu, "cpu", "100%", "CPU cap")
-	flag.StringVar(&config.Root, "root", config.DefaultRoot, "Root path to default site")
-	flag.StringVar(&config.Host, "host", config.DefaultHost, "Default host")
-	flag.StringVar(&config.Port, "port", config.DefaultPort, "Default port")
+	flag.StringVar(&caddy.Root, "root", caddy.DefaultRoot, "Root path to default site")
+	flag.StringVar(&caddy.Host, "host", caddy.DefaultHost, "Default host")
+	flag.StringVar(&caddy.Port, "port", caddy.DefaultPort, "Default port")
 	flag.BoolVar(&version, "version", false, "Show version")
+	// TODO: Boulder dev URL is: http://192.168.99.100:4000
+	// TODO: Staging API URL is: https://acme-staging.api.letsencrypt.org
+	// TODO: Production endpoint is: https://acme-v01.api.letsencrypt.org
+	flag.StringVar(&letsencrypt.CAUrl, "ca", "https://acme-staging.api.letsencrypt.org", "Certificate authority ACME server")
 	flag.BoolVar(&letsencrypt.Agreed, "agree", false, "Agree to Let's Encrypt Subscriber Agreement")
 	flag.StringVar(&letsencrypt.DefaultEmail, "email", "", "Default email address to use for Let's Encrypt transactions")
 	flag.StringVar(&revoke, "revoke", "", "Hostname for which to revoke the certificate")
@@ -43,8 +48,11 @@ func init() {
 func main() {
 	flag.Parse()
 
+	caddy.AppName = appName
+	caddy.AppVersion = appVersion
+
 	if version {
-		fmt.Printf("%s %s\n", app.Name, app.Version)
+		fmt.Printf("%s %s\n", caddy.AppName, caddy.AppVersion)
 		os.Exit(0)
 	}
 	if revoke != "" {
@@ -57,131 +65,96 @@ func main() {
 	}
 
 	// Set CPU cap
-	err := app.SetCPU(cpu)
+	err := setCPU(cpu)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	// Load config from file
-	addresses, err := loadConfigs()
+	// Get Caddyfile input
+	caddyfile, err := caddy.LoadCaddyfile(loadCaddyfile)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	// Start each server with its one or more configurations
-	for addr, configs := range addresses {
-		s, err := server.New(addr.String(), configs)
-		if err != nil {
-			log.Fatal(err)
-		}
-		s.HTTP2 = app.HTTP2 // TODO: This setting is temporary
-		app.Wg.Add(1)
-		go func(s *server.Server) {
-			defer app.Wg.Done()
-			err := s.Serve()
-			if err != nil {
-				log.Fatal(err) // kill whole process to avoid a half-alive zombie server
-			}
-		}(s)
-
-		app.Servers = append(app.Servers, s)
-	}
-
-	// Show initialization output
-	if !app.Quiet {
-		var checkedFdLimit bool
-		for addr, configs := range addresses {
-			for _, conf := range configs {
-				// Print address of site
-				fmt.Println(conf.Address())
-
-				// Note if non-localhost site resolves to loopback interface
-				if addr.IP.IsLoopback() && !isLocalhost(conf.Host) {
-					fmt.Printf("Notice: %s is only accessible on this machine (%s)\n",
-						conf.Host, addr.IP.String())
-				}
-				if !checkedFdLimit && !addr.IP.IsLoopback() && !isLocalhost(conf.Host) {
-					checkFdlimit()
-					checkedFdLimit = true
-				}
-			}
-		}
-	}
-
-	// Wait for all listeners to stop
-	app.Wg.Wait()
-}
-
-// checkFdlimit issues a warning if the OS max file descriptors is below a recommended minimum.
-func checkFdlimit() {
-	const min = 4096
-
-	// Warn if ulimit is too low for production sites
-	if runtime.GOOS == "linux" || runtime.GOOS == "darwin" {
-		out, err := exec.Command("sh", "-c", "ulimit -n").Output() // use sh because ulimit isn't in Linux $PATH
-		if err == nil {
-			// Note that an error here need not be reported
-			lim, err := strconv.Atoi(string(bytes.TrimSpace(out)))
-			if err == nil && lim < min {
-				fmt.Printf("Warning: File descriptor limit %d is too low for production sites. At least %d is recommended. Set with \"ulimit -n %d\".\n", lim, min, min)
-			}
-		}
+	// Start your engines
+	err = caddy.Start(caddyfile)
+	if err != nil {
+		log.Fatal(err)
 	}
-}
 
-// isLocalhost returns true if the string looks explicitly like a localhost address.
-func isLocalhost(s string) bool {
-	return s == "localhost" || s == "::1" || strings.HasPrefix(s, "127.")
+	// Twiddle your thumbs
+	caddy.Wait()
 }
 
-// loadConfigs loads configuration from a file or stdin (piped).
-// The configurations are grouped by bind address.
-// Configuration is obtained from one of four sources, tried
-// in this order: 1. -conf flag, 2. stdin, 3. command line argument 4. Caddyfile.
-// If none of those are available, a default configuration is loaded.
-func loadConfigs() (config.Group, error) {
+func loadCaddyfile() (caddy.Input, error) {
 	// -conf flag
 	if conf != "" {
-		file, err := os.Open(conf)
+		contents, err := ioutil.ReadFile(conf)
 		if err != nil {
 			return nil, err
 		}
-		defer file.Close()
-		return config.Load(path.Base(conf), file)
-	}
-
-	// stdin
-	fi, err := os.Stdin.Stat()
-	if err == nil && fi.Mode()&os.ModeCharDevice == 0 {
-		// Note that a non-nil error is not a problem. Windows
-		// will not create a stdin if there is no pipe, which
-		// produces an error when calling Stat(). But Unix will
-		// make one either way, which is why we also check that
-		// bitmask.
-		confBody, err := ioutil.ReadAll(os.Stdin)
-		if err != nil {
-			log.Fatal(err)
-		}
-		if len(confBody) > 0 {
-			return config.Load("stdin", bytes.NewReader(confBody))
-		}
+		return caddy.CaddyfileInput{
+			Contents: contents,
+			Filepath: conf,
+			RealFile: true,
+		}, nil
 	}
 
-	// Command line Arg
+	// command line args
 	if flag.NArg() > 0 {
-		confBody := ":" + config.DefaultPort + "\n" + strings.Join(flag.Args(), "\n")
-		return config.Load("args", bytes.NewBufferString(confBody))
+		confBody := ":" + caddy.DefaultPort + "\n" + strings.Join(flag.Args(), "\n")
+		return caddy.CaddyfileInput{
+			Contents: []byte(confBody),
+			Filepath: "args",
+		}, nil
 	}
 
-	// Caddyfile
-	file, err := os.Open(config.DefaultConfigFile)
+	// Caddyfile in cwd
+	contents, err := ioutil.ReadFile(caddy.DefaultConfigFile)
 	if err != nil {
 		if os.IsNotExist(err) {
-			return config.Default()
+			return caddy.DefaultInput, nil
 		}
 		return nil, err
 	}
-	defer file.Close()
+	return caddy.CaddyfileInput{
+		Contents: contents,
+		Filepath: caddy.DefaultConfigFile,
+		RealFile: true,
+	}, nil
+}
+
+// setCPU parses string cpu and sets GOMAXPROCS
+// according to its value. It accepts either
+// a number (e.g. 3) or a percent (e.g. 50%).
+func setCPU(cpu string) error {
+	var numCPU int
+
+	availCPU := runtime.NumCPU()
+
+	if strings.HasSuffix(cpu, "%") {
+		// Percent
+		var percent float32
+		pctStr := cpu[:len(cpu)-1]
+		pctInt, err := strconv.Atoi(pctStr)
+		if err != nil || pctInt < 1 || pctInt > 100 {
+			return errors.New("invalid CPU value: percentage must be between 1-100")
+		}
+		percent = float32(pctInt) / 100
+		numCPU = int(float32(availCPU) * percent)
+	} else {
+		// Number
+		num, err := strconv.Atoi(cpu)
+		if err != nil || num < 1 {
+			return errors.New("invalid CPU value: provide a number or percent greater than 0")
+		}
+		numCPU = num
+	}
+
+	if numCPU > availCPU {
+		numCPU = availCPU
+	}
 
-	return config.Load(config.DefaultConfigFile, file)
+	runtime.GOMAXPROCS(numCPU)
+	return nil
 }

middleware/proxy/upstream.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/mholt/caddy/config/parse"
+	"github.com/mholt/caddy/caddy/parse"
 )
 
 var (

server/graceful.go

+package server
+
+import (
+	"net"
+	"os"
+	"sync"
+	"syscall"
+)
+
+// newGracefulListener returns a gracefulListener that wraps l and
+// uses wg (stored in the host server) to count connections.
+func newGracefulListener(l ListenerFile, wg *sync.WaitGroup) *gracefulListener {
+	gl := &gracefulListener{ListenerFile: l, stop: make(chan error), httpWg: wg}
+	go func() {
+		<-gl.stop
+		gl.stopped = true
+		gl.stop <- gl.ListenerFile.Close()
+	}()
+	return gl
+}
+
+// gracefuListener is a net.Listener which can
+// count the number of connections on it. Its
+// methods mainly wrap net.Listener to be graceful.
+type gracefulListener struct {
+	ListenerFile
+	stop    chan error
+	stopped bool
+	httpWg  *sync.WaitGroup // pointer to the host's wg used for counting connections
+}
+
+// Accept accepts a connection. This type wraps
+func (gl *gracefulListener) Accept() (c net.Conn, err error) {
+	c, err = gl.ListenerFile.Accept()
+	if err != nil {
+		return
+	}
+	c = gracefulConn{Conn: c, httpWg: gl.httpWg}
+	gl.httpWg.Add(1)
+	return
+}
+
+// Close immediately closes the listener.
+func (gl *gracefulListener) Close() error {
+	if gl.stopped {
+		return syscall.EINVAL
+	}
+	gl.stop <- nil
+	return <-gl.stop
+}
+
+// File implements ListenerFile; it gets the file of the listening socket.
+func (gl *gracefulListener) File() (*os.File, error) {
+	return gl.ListenerFile.File()
+}
+
+// gracefulConn represents a connection on a
+// gracefulListener so that we can keep track
+// of the number of connections, thus facilitating
+// a graceful shutdown.
+type gracefulConn struct {
+	net.Conn
+	httpWg *sync.WaitGroup // pointer to the host server's connection waitgroup
+}
+
+// Close closes c's underlying connection while updating the wg count.
+func (c gracefulConn) Close() error {
+	err := c.Conn.Close()
+	if err != nil {
+		return err
+	}
+	// close can fail on http2 connections (as of Oct. 2015, before http2 in std lib)
+	// so don't decrement count unless close succeeds
+	c.httpWg.Done()
+	return nil
+}

server/server.go

@@ -12,18 +12,31 @@ import (
 	"net"
 	"net/http"
 	"os"
-	"os/signal"
+	"runtime"
+	"sync"
+	"time"
 
 	"golang.org/x/net/http2"
 )
 
 // Server represents an instance of a server, which serves
-// static content at a particular address (host and port).
+// HTTP requests at a particular address (host and port). A
+// server is capable of serving numerous virtual hosts on
+// the same address and the listener may be stopped for
+// graceful termination (POSIX only).
 type Server struct {
-	HTTP2   bool                   // temporary while http2 is not in std lib (TODO: remove flag when part of std lib)
-	address string                 // the actual address for net.Listen to listen on
-	tls     bool                   // whether this server is serving all HTTPS hosts or not
-	vhosts  map[string]virtualHost // virtual hosts keyed by their address
+	*http.Server
+	HTTP2      bool                   // temporary while http2 is not in std lib (TODO: remove flag when part of std lib)
+	tls        bool                   // whether this server is serving all HTTPS hosts or not
+	vhosts     map[string]virtualHost // virtual hosts keyed by their address
+	listener   ListenerFile           // the listener which is bound to the socket
+	listenerMu sync.Mutex             // protects listener
+	httpWg     sync.WaitGroup         // used to wait on outstanding connections
+}
+
+type ListenerFile interface {
+	net.Listener
+	File() (*os.File, error)
 }
 
 // New creates a new Server which will bind to addr and serve
@@ -36,14 +49,29 @@ func New(addr string, configs []Config) (*Server, error) {
 	}
 
 	s := &Server{
-		address: addr,
-		tls:     tls,
-		vhosts:  make(map[string]virtualHost),
+		Server: &http.Server{
+			Addr: addr,
+			// TODO: Make these values configurable?
+			// ReadTimeout:    2 * time.Minute,
+			// WriteTimeout:   2 * time.Minute,
+			// MaxHeaderBytes: 1 << 16,
+		},
+		tls:    tls,
+		vhosts: make(map[string]virtualHost),
 	}
+	s.Handler = s // this is weird, but whatever
+
+	// We have to bound our wg with one increment
+	// to prevent a "race condition" that is hard-coded
+	// into sync.WaitGroup.Wait() - basically, an add
+	// with a positive delta must be guaranteed to
+	// occur before Wait() is called on the wg.
+	s.httpWg.Add(1)
 
+	// Set up each virtualhost
 	for _, conf := range configs {
 		if _, exists := s.vhosts[conf.Host]; exists {
-			return nil, fmt.Errorf("cannot serve %s - host already defined for address %s", conf.Address(), s.address)
+			return nil, fmt.Errorf("cannot serve %s - host already defined for address %s", conf.Address(), s.Addr)
 		}
 
 		vh := virtualHost{config: conf}
@@ -60,98 +88,87 @@ func New(addr string, configs []Config) (*Server, error) {
 	return s, nil
 }
 
-// Serve starts the server. It blocks until the server quits.
-func (s *Server) Serve() error {
-	server := &http.Server{
-		Addr:    s.address,
-		Handler: s,
+// Serve starts the server with an existing listener. It blocks until the
+// server stops.
+func (s *Server) Serve(ln ListenerFile) error {
+	err := s.setup()
+	if err != nil {
+		return err
 	}
+	return s.serve(ln)
+}
 
-	if s.HTTP2 {
-		// TODO: This call may not be necessary after HTTP/2 is merged into std lib
-		http2.ConfigureServer(server, nil)
+// ListenAndServe starts the server with a new listener. It blocks until the server stops.
+func (s *Server) ListenAndServe() error {
+	err := s.setup()
+	if err != nil {
+		return err
 	}
 
-	for _, vh := range s.vhosts {
-		// Execute startup functions now
-		for _, start := range vh.config.Startup {
-			err := start()
-			if err != nil {
-				return err
-			}
-		}
+	ln, err := net.Listen("tcp", s.Addr)
+	if err != nil {
+		return err
+	}
 
-		// Execute shutdown commands on exit
-		if len(vh.config.Shutdown) > 0 {
-			go func(vh virtualHost) {
-				// Wait for signal
-				interrupt := make(chan os.Signal, 1)
-				signal.Notify(interrupt, os.Interrupt, os.Kill) // TODO: syscall.SIGQUIT? (Ctrl+\, Unix-only)
-				<-interrupt
-
-				// Run callbacks
-				exitCode := 0
-				for _, shutdownFunc := range vh.config.Shutdown {
-					err := shutdownFunc()
-					if err != nil {
-						exitCode = 1
-						log.Println(err)
-					}
-				}
-				os.Exit(exitCode) // BUG: Other shutdown goroutines might be running; use sync.WaitGroup
-			}(vh)
-		}
+	return s.serve(ln.(*net.TCPListener))
+}
+
+// serve prepares s to listen on ln by wrapping ln in a
+// tcpKeepAliveListener (if ln is a *net.TCPListener) and
+// then in a gracefulListener, so that keep-alive is supported
+// as well as graceful shutdown/restart. It also configures
+// TLS listener on top of that if applicable.
+func (s *Server) serve(ln ListenerFile) error {
+	if tcpLn, ok := ln.(*net.TCPListener); ok {
+		ln = tcpKeepAliveListener{TCPListener: tcpLn}
 	}
 
+	s.listenerMu.Lock()
+	s.listener = newGracefulListener(ln, &s.httpWg)
+	s.listenerMu.Unlock()
+
 	if s.tls {
 		var tlsConfigs []TLSConfig
 		for _, vh := range s.vhosts {
 			tlsConfigs = append(tlsConfigs, vh.config.TLS)
 		}
-		return ListenAndServeTLSWithSNI(server, tlsConfigs)
+		return serveTLSWithSNI(s, s.listener, tlsConfigs)
 	}
-	return server.ListenAndServe()
+
+	return s.Server.Serve(s.listener)
 }
 
-// copy from net/http/transport.go
-func cloneTLSConfig(cfg *tls.Config) *tls.Config {
-	if cfg == nil {
-		return &tls.Config{}
-	}
-	return &tls.Config{
-		Rand:                     cfg.Rand,
-		Time:                     cfg.Time,
-		Certificates:             cfg.Certificates,
-		NameToCertificate:        cfg.NameToCertificate,
-		GetCertificate:           cfg.GetCertificate,
-		RootCAs:                  cfg.RootCAs,
-		NextProtos:               cfg.NextProtos,
-		ServerName:               cfg.ServerName,
-		ClientAuth:               cfg.ClientAuth,
-		ClientCAs:                cfg.ClientCAs,
-		InsecureSkipVerify:       cfg.InsecureSkipVerify,
-		CipherSuites:             cfg.CipherSuites,
-		PreferServerCipherSuites: cfg.PreferServerCipherSuites,
-		SessionTicketsDisabled:   cfg.SessionTicketsDisabled,
-		SessionTicketKey:         cfg.SessionTicketKey,
-		ClientSessionCache:       cfg.ClientSessionCache,
-		MinVersion:               cfg.MinVersion,
-		MaxVersion:               cfg.MaxVersion,
-		CurvePreferences:         cfg.CurvePreferences,
+// setup prepares the server s to begin listening; it should be
+// called just before the listener announces itself on the network
+// and should only be called when the server is just starting up.
+func (s *Server) setup() error {
+	if s.HTTP2 {
+		// TODO: This call may not be necessary after HTTP/2 is merged into std lib
+		http2.ConfigureServer(s.Server, nil)
 	}
-}
 
-// ListenAndServeTLSWithSNI serves TLS with Server Name Indication (SNI) support, which allows
-// multiple sites (different hostnames) to be served from the same address. This method is
-// adapted directly from the std lib's net/http ListenAndServeTLS function, which was
-// written by the Go Authors. It has been modified to support multiple certificate/key pairs.
-func ListenAndServeTLSWithSNI(srv *http.Server, tlsConfigs []TLSConfig) error {
-	addr := srv.Addr
-	if addr == "" {
-		addr = ":https"
+	// Execute startup functions now
+	for _, vh := range s.vhosts {
+		for _, startupFunc := range vh.config.Startup {
+			err := startupFunc()
+			if err != nil {
+				return err
+			}
+		}
 	}
 
-	config := cloneTLSConfig(srv.TLSConfig)
+	return nil
+}
+
+// serveTLSWithSNI serves TLS with Server Name Indication (SNI) support, which allows
+// multiple sites (different hostnames) to be served from the same address. It also
+// supports client authentication if srv has it enabled. It blocks until s quits.
+//
+// This method is adapted from the std lib's net/http ServeTLS function, which was written
+// by the Go Authors. It has been modified to support multiple certificate/key pairs,
+// client authentication, and our custom Server type.
+func serveTLSWithSNI(s *Server, ln net.Listener, tlsConfigs []TLSConfig) error {
+	config := cloneTLSConfig(s.TLSConfig)
 	if config.NextProtos == nil {
 		config.NextProtos = []string{"http/1.1"}
 	}
@@ -181,45 +198,62 @@ func ListenAndServeTLSWithSNI(srv *http.Server, tlsConfigs []TLSConfig) error {
 		return err
 	}
 
-	// Create listener and we're on our way
-	conn, err := net.Listen("tcp", addr)
-	if err != nil {
-		return err
-	}
-	tlsListener := tls.NewListener(conn, config)
+	// Create TLS listener - note that we do not replace s.listener
+	// with this TLS listener; tls.listener is unexported and does
+	// not implement the File() method we need for graceful restarts
+	// on POSIX systems.
+	ln = tls.NewListener(ln, config)
 
-	return srv.Serve(tlsListener)
+	// Begin serving; block until done
+	return s.Server.Serve(ln)
 }
 
-// setupClientAuth sets up TLS client authentication only if
-// any of the TLS configs specified at least one cert file.
-func setupClientAuth(tlsConfigs []TLSConfig, config *tls.Config) error {
-	var clientAuth bool
-	for _, cfg := range tlsConfigs {
-		if len(cfg.ClientCerts) > 0 {
-			clientAuth = true
-			break
+// Stop stops the server. It blocks until the server is
+// totally stopped. On POSIX systems, it will wait for
+// connections to close (up to a max timeout of a few
+// seconds); on Windows it will close the listener
+// immediately.
+func (s *Server) Stop() error {
+	s.Server.SetKeepAlivesEnabled(false) // TODO: Does this even do anything? :P
+
+	if runtime.GOOS != "windows" {
+		// force connections to close after timeout
+		done := make(chan struct{})
+		go func() {
+			s.httpWg.Done() // decrement our initial increment used as a barrier
+			s.httpWg.Wait()
+			close(done)
+		}()
+
+		// Wait for remaining connections to finish or
+		// force them all to close after timeout
+		select {
+		case <-time.After(5 * time.Second): // TODO: configurable?
+		case <-done:
 		}
 	}
 
-	if clientAuth {
-		pool := x509.NewCertPool()
-		for _, cfg := range tlsConfigs {
-			for _, caFile := range cfg.ClientCerts {
-				caCrt, err := ioutil.ReadFile(caFile) // Anyone that gets a cert from Matt Holt can connect
-				if err != nil {
-					return err
-				}
-				if !pool.AppendCertsFromPEM(caCrt) {
-					return fmt.Errorf("error loading client certificate '%s': no certificates were successfully parsed", caFile)
-				}
-			}
-		}
-		config.ClientCAs = pool
-		config.ClientAuth = tls.RequireAndVerifyClientCert
+	// Close the listener now; this stops the server and
+	s.listenerMu.Lock()
+	err := s.listener.Close()
+	s.listenerMu.Unlock()
+	if err != nil {
+		// TODO: Better logging
+		log.Println(err)
 	}
 
-	return nil
+	return err
+}
+
+// ListenerFd gets the file descriptor of the listener.
+func (s *Server) ListenerFd() uintptr {
+	s.listenerMu.Lock()
+	defer s.listenerMu.Unlock()
+	file, err := s.listener.File()
+	if err != nil {
+		return 0
+	}
+	return file.Fd()
 }
 
 // ServeHTTP is the entry point for every request to the address that s
@@ -261,7 +295,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		}
 	} else {
 		w.WriteHeader(http.StatusNotFound)
-		fmt.Fprintf(w, "No such host at %s", s.address)
+		fmt.Fprintf(w, "No such host at %s", s.Server.Addr)
 	}
 }
 
@@ -271,3 +305,110 @@ func DefaultErrorFunc(w http.ResponseWriter, r *http.Request, status int) {
 	w.WriteHeader(status)
 	fmt.Fprintf(w, "%d %s", status, http.StatusText(status))
 }
+
+// setupClientAuth sets up TLS client authentication only if
+// any of the TLS configs specified at least one cert file.
+func setupClientAuth(tlsConfigs []TLSConfig, config *tls.Config) error {
+	var clientAuth bool
+	for _, cfg := range tlsConfigs {
+		if len(cfg.ClientCerts) > 0 {
+			clientAuth = true
+			break
+		}
+	}
+
+	if clientAuth {
+		pool := x509.NewCertPool()
+		for _, cfg := range tlsConfigs {
+			for _, caFile := range cfg.ClientCerts {
+				caCrt, err := ioutil.ReadFile(caFile) // Anyone that gets a cert from this CA can connect
+				if err != nil {
+					return err
+				}
+				if !pool.AppendCertsFromPEM(caCrt) {
+					return fmt.Errorf("error loading client certificate '%s': no certificates were successfully parsed", caFile)
+				}
+			}
+		}
+		config.ClientCAs = pool
+		config.ClientAuth = tls.RequireAndVerifyClientCert
+	}
+
+	return nil
+}
+
+// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted
+// connections. It's used by ListenAndServe and ListenAndServeTLS so
+// dead TCP connections (e.g. closing laptop mid-download) eventually
+// go away.
+//
+// Borrowed from the Go standard library.
+type tcpKeepAliveListener struct {
+	*net.TCPListener
+}
+
+// Accept accepts the connection with a keep-alive enabled.
+func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) {
+	tc, err := ln.AcceptTCP()
+	if err != nil {
+		return
+	}
+	tc.SetKeepAlive(true)
+	tc.SetKeepAlivePeriod(3 * time.Minute)
+	return tc, nil
+}
+
+// File implements ListenerFile; returns the underlying file of the listener.
+func (ln tcpKeepAliveListener) File() (*os.File, error) {
+	return ln.TCPListener.File()
+}
+
+// copied from net/http/transport.go
+func cloneTLSConfig(cfg *tls.Config) *tls.Config {
+	if cfg == nil {
+		return &tls.Config{}
+	}
+	return &tls.Config{
+		Rand:                     cfg.Rand,
+		Time:                     cfg.Time,
+		Certificates:             cfg.Certificates,
+		NameToCertificate:        cfg.NameToCertificate,
+		GetCertificate:           cfg.GetCertificate,
+		RootCAs:                  cfg.RootCAs,
+		NextProtos:               cfg.NextProtos,
+		ServerName:               cfg.ServerName,
+		ClientAuth:               cfg.ClientAuth,
+		ClientCAs:                cfg.ClientCAs,
+		InsecureSkipVerify:       cfg.InsecureSkipVerify,
+		CipherSuites:             cfg.CipherSuites,
+		PreferServerCipherSuites: cfg.PreferServerCipherSuites,
+		SessionTicketsDisabled:   cfg.SessionTicketsDisabled,
+		SessionTicketKey:         cfg.SessionTicketKey,
+		ClientSessionCache:       cfg.ClientSessionCache,
+		MinVersion:               cfg.MinVersion,
+		MaxVersion:               cfg.MaxVersion,
+		CurvePreferences:         cfg.CurvePreferences,
+	}
+}
+
+// ShutdownCallbacks executes all the shutdown callbacks
+// for all the virtualhosts in servers, and returns all the
+// errors generated during their execution. In other words,
+// an error executing one shutdown callback does not stop
+// execution of others. Only one shutdown callback is executed
+// at a time. You must protect the servers that are passed in
+// if they are shared across threads.
+func ShutdownCallbacks(servers []*Server) []error {
+	var errs []error
+	for _, s := range servers {
+		for _, vhost := range s.vhosts {
+			for _, shutdownFunc := range vhost.config.Shutdown {
+				err := shutdownFunc()
+				if err != nil {
+					errs = append(errs, err)
+				}
+			}
+		}
+	}
+	return errs
+}