letsencrypt: Graceful restarts

Lots of refinement still needed and runs only on POSIX systems. Windows will not get true graceful restarts (for now), but we will opt for very, very quick forceful restarts. Also, server configs are no longer put into a map; it is critical that they stay ordered so that they can be matched with their sockets in the child process after forking.

This implementation of graceful restarts is probably not perfect, but it is a good start. Lots of details to attend to now.

app/app.go

@@ -7,12 +7,15 @@ package app
 
 import (
 	"errors"
+	"log"
 	"os"
+	"os/signal"
 	"path/filepath"
 	"runtime"
 	"strconv"
 	"strings"
 	"sync"
+	"syscall"
 
 	"github.com/mholt/caddy/server"
 )
@@ -42,6 +45,75 @@ var (
 	Quiet bool
 )
 
+func init() {
+	go func() {
+		// Wait for signal
+		interrupt := make(chan os.Signal, 1)
+		signal.Notify(interrupt, os.Interrupt, os.Kill) // TODO: syscall.SIGTERM? Or that should not run callbacks...
+		<-interrupt
+
+		// Run shutdown callbacks
+		var exitCode int
+		ServersMutex.Lock()
+		errs := server.ShutdownCallbacks(Servers)
+		ServersMutex.Unlock()
+		if len(errs) > 0 {
+			for _, err := range errs {
+				log.Println(err)
+			}
+			exitCode = 1
+		}
+		os.Exit(exitCode)
+	}()
+}
+
+// Restart restarts the entire application; gracefully with zero
+// downtime if on a POSIX-compatible system, or forcefully if on
+// Windows but with imperceptibly-short downtime.
+//
+// The restarted application will use caddyfile as its input
+// configuration; it will not look elsewhere for the config
+// to use.
+func Restart(caddyfile []byte) error {
+	// TODO: This is POSIX-only right now; also, os.Args[0] is required!
+	// TODO: Pipe the Caddyfile to stdin of child!
+	// TODO: Before stopping this process, verify child started successfully (valid Caddyfile, etc)
+
+	// Tell the child that it's a restart
+	os.Setenv("CADDY_RESTART", "true")
+
+	// Pass along current environment and file descriptors to child.
+	// We pass along the file descriptors explicitly to ensure proper
+	// order, since losing the original order will break the child.
+	fds := []uintptr{os.Stdin.Fd(), os.Stdout.Fd(), os.Stderr.Fd()}
+
+	// Now add file descriptors of the sockets
+	ServersMutex.Lock()
+	for _, s := range Servers {
+		fds = append(fds, s.ListenerFd())
+	}
+	ServersMutex.Unlock()
+
+	// Fork the process with the current environment and file descriptors
+	execSpec := &syscall.ProcAttr{
+		Env:   os.Environ(),
+		Files: fds,
+	}
+	fork, err := syscall.ForkExec(os.Args[0], os.Args, execSpec)
+	if err != nil {
+		log.Println("FORK ERR:", err, fork)
+	}
+
+	// Child process is listening now; we can stop all our servers here.
+	ServersMutex.Lock()
+	for _, s := range Servers {
+		go s.Stop() // TODO: error checking/reporting
+	}
+	ServersMutex.Unlock()
+
+	return err
+}
+
 // SetCPU parses string cpu and sets GOMAXPROCS
 // according to its value. It accepts either
 // a number (e.g. 3) or a percent (e.g. 50%).

config/config.go

@@ -153,14 +153,14 @@ func makeStorages() map[string]interface{} {
 // bind address to list of configs that would become VirtualHosts on that
 // server. Use the keys of the returned map to create listeners, and use
 // the associated values to set up the virtualhosts.
-func arrangeBindings(allConfigs []server.Config) (map[*net.TCPAddr][]server.Config, error) {
-	addresses := make(map[*net.TCPAddr][]server.Config)
+func arrangeBindings(allConfigs []server.Config) (Group, error) {
+	var groupings Group
 
 	// Group configs by bind address
 	for _, conf := range allConfigs {
-		newAddr, warnErr, fatalErr := resolveAddr(conf)
+		bindAddr, warnErr, fatalErr := resolveAddr(conf)
 		if fatalErr != nil {
-			return addresses, fatalErr
+			return groupings, fatalErr
 		}
 		if warnErr != nil {
 			log.Println("[Warning]", warnErr)
@@ -169,37 +169,40 @@ func arrangeBindings(allConfigs []server.Config) (map[*net.TCPAddr][]server.Conf
 		// Make sure to compare the string representation of the address,
 		// not the pointer, since a new *TCPAddr is created each time.
 		var existing bool
-		for addr := range addresses {
-			if addr.String() == newAddr.String() {
-				addresses[addr] = append(addresses[addr], conf)
+		for i := 0; i < len(groupings); i++ {
+			if groupings[i].BindAddr.String() == bindAddr.String() {
+				groupings[i].Configs = append(groupings[i].Configs, conf)
 				existing = true
 				break
 			}
 		}
 		if !existing {
-			addresses[newAddr] = append(addresses[newAddr], conf)
+			groupings = append(groupings, BindingMapping{
+				BindAddr: bindAddr,
+				Configs:  []server.Config{conf},
+			})
 		}
 	}
 
 	// Don't allow HTTP and HTTPS to be served on the same address
-	for _, configs := range addresses {
-		isTLS := configs[0].TLS.Enabled
-		for _, config := range configs {
+	for _, group := range groupings {
+		isTLS := group.Configs[0].TLS.Enabled
+		for _, config := range group.Configs {
 			if config.TLS.Enabled != isTLS {
 				thisConfigProto, otherConfigProto := "HTTP", "HTTP"
 				if config.TLS.Enabled {
 					thisConfigProto = "HTTPS"
 				}
-				if configs[0].TLS.Enabled {
+				if group.Configs[0].TLS.Enabled {
 					otherConfigProto = "HTTPS"
 				}
-				return addresses, fmt.Errorf("configuration error: Cannot multiplex %s (%s) and %s (%s) on same address",
-					configs[0].Address(), otherConfigProto, config.Address(), thisConfigProto)
+				return groupings, fmt.Errorf("configuration error: Cannot multiplex %s (%s) and %s (%s) on same address",
+					group.Configs[0].Address(), otherConfigProto, config.Address(), thisConfigProto)
 			}
 		}
 	}
 
-	return addresses, nil
+	return groupings, nil
 }
 
 // resolveAddr determines the address (host and port) that a config will
@@ -291,5 +294,15 @@ var (
 	Port = DefaultPort
 )
 
+// BindingMapping maps a network address to configurations
+// that will bind to it. The order of the configs is important.
+type BindingMapping struct {
+	BindAddr *net.TCPAddr
+	Configs  []server.Config
+}
+
 // Group maps network addresses to their configurations.
-type Group map[*net.TCPAddr][]server.Config
+// Preserving the order of the groupings is important
+// (related to graceful shutdown and restart)
+// so this is a slice, not a literal map.
+type Group []BindingMapping

main.go

@@ -6,12 +6,14 @@ import (
 	"fmt"
 	"io/ioutil"
 	"log"
+	"net"
 	"os"
 	"os/exec"
 	"path"
 	"runtime"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/mholt/caddy/app"
 	"github.com/mholt/caddy/config"
@@ -63,44 +65,70 @@ func main() {
 	}
 
 	// Load config from file
-	addresses, err := loadConfigs()
+	groupings, err := loadConfigs()
 	if err != nil {
 		log.Fatal(err)
 	}
 
 	// Start each server with its one or more configurations
-	for addr, configs := range addresses {
-		s, err := server.New(addr.String(), configs)
+	for i, group := range groupings {
+		s, err := server.New(group.BindAddr.String(), group.Configs)
 		if err != nil {
 			log.Fatal(err)
 		}
 		s.HTTP2 = app.HTTP2 // TODO: This setting is temporary
+
 		app.Wg.Add(1)
-		go func(s *server.Server) {
+		go func(s *server.Server, i int) {
 			defer app.Wg.Done()
-			err := s.Serve()
+
+			if os.Getenv("CADDY_RESTART") == "true" {
+				file := os.NewFile(uintptr(3+i), "")
+				ln, err := net.FileListener(file)
+				if err != nil {
+					log.Fatal("FILE LISTENER:", err)
+				}
+
+				var ok bool
+				ln, ok = ln.(server.ListenerFile)
+				if !ok {
+					log.Fatal("Listener was not a ListenerFile")
+				}
+
+				err = s.Serve(ln.(server.ListenerFile))
+				// TODO: Better error logging... also, is it even necessary?
+				if err != nil {
+					log.Println(err)
+				}
+			} else {
+				err := s.ListenAndServe()
+				// TODO: Better error logging... also, is it even necessary?
+				// For example, "use of closed network connection" is normal if doing graceful shutdown...
 				if err != nil {
-				log.Fatal(err) // kill whole process to avoid a half-alive zombie server
+					log.Println(err)
 				}
-		}(s)
+			}
+		}(s, i)
 
+		app.ServersMutex.Lock()
 		app.Servers = append(app.Servers, s)
+		app.ServersMutex.Unlock()
 	}
 
 	// Show initialization output
 	if !app.Quiet {
 		var checkedFdLimit bool
-		for addr, configs := range addresses {
-			for _, conf := range configs {
+		for _, group := range groupings {
+			for _, conf := range group.Configs {
 				// Print address of site
 				fmt.Println(conf.Address())
 
 				// Note if non-localhost site resolves to loopback interface
-				if addr.IP.IsLoopback() && !isLocalhost(conf.Host) {
+				if group.BindAddr.IP.IsLoopback() && !isLocalhost(conf.Host) {
 					fmt.Printf("Notice: %s is only accessible on this machine (%s)\n",
-						conf.Host, addr.IP.String())
+						conf.Host, group.BindAddr.IP.String())
 				}
-				if !checkedFdLimit && !addr.IP.IsLoopback() && !isLocalhost(conf.Host) {
+				if !checkedFdLimit && !group.BindAddr.IP.IsLoopback() && !isLocalhost(conf.Host) {
 					checkFdlimit()
 					checkedFdLimit = true
 				}
@@ -108,7 +136,16 @@ func main() {
 		}
 	}
 
-	// Wait for all listeners to stop
+	// TODO: Temporary; testing restart
+	if os.Getenv("CADDY_RESTART") != "true" {
+		go func() {
+			time.Sleep(5 * time.Second)
+			fmt.Println("restarting")
+			log.Println("RESTART ERR:", app.Restart([]byte{}))
+		}()
+	}
+
+	// Wait for all servers to be stopped
 	app.Wg.Wait()
 }
 

server/graceful.go

+package server
+
+import (
+	"net"
+	"os"
+	"sync"
+	"syscall"
+)
+
+// newGracefulListener returns a gracefulListener that wraps l and
+// uses wg (stored in the host server) to count connections.
+func newGracefulListener(l ListenerFile, wg *sync.WaitGroup) *gracefulListener {
+	gl := &gracefulListener{ListenerFile: l, stop: make(chan error), httpWg: wg}
+	go func() {
+		<-gl.stop
+		gl.stopped = true
+		gl.stop <- gl.ListenerFile.Close()
+	}()
+	return gl
+}
+
+// gracefuListener is a net.Listener which can
+// count the number of connections on it. Its
+// methods mainly wrap net.Listener to be graceful.
+type gracefulListener struct {
+	ListenerFile
+	stop    chan error
+	stopped bool
+	httpWg  *sync.WaitGroup // pointer to the host's wg used for counting connections
+}
+
+// Accept accepts a connection. This type wraps
+func (gl *gracefulListener) Accept() (c net.Conn, err error) {
+	c, err = gl.ListenerFile.Accept()
+	if err != nil {
+		return
+	}
+	c = gracefulConn{Conn: c, httpWg: gl.httpWg}
+	gl.httpWg.Add(1)
+	return
+}
+
+// Close immediately closes the listener.
+func (gl *gracefulListener) Close() error {
+	if gl.stopped {
+		return syscall.EINVAL
+	}
+	gl.stop <- nil
+	return <-gl.stop
+}
+
+// File implements ListenerFile; it gets the file of the listening socket.
+func (gl *gracefulListener) File() (*os.File, error) {
+	return gl.ListenerFile.File()
+}
+
+// gracefulConn represents a connection on a
+// gracefulListener so that we can keep track
+// of the number of connections, thus facilitating
+// a graceful shutdown.
+type gracefulConn struct {
+	net.Conn
+	httpWg *sync.WaitGroup // pointer to the host server's connection waitgroup
+}
+
+// Close closes c's underlying connection while updating the wg count.
+func (c gracefulConn) Close() error {
+	c.httpWg.Done()
+	return c.Conn.Close()
+}

server/server.go

@@ -12,18 +12,31 @@ import (
 	"net"
 	"net/http"
 	"os"
-	"os/signal"
+	"runtime"
+	"sync"
+	"time"
 
 	"golang.org/x/net/http2"
 )
 
 // Server represents an instance of a server, which serves
-// static content at a particular address (host and port).
+// HTTP requests at a particular address (host and port). A
+// server is capable of serving numerous virtual hosts on
+// the same address and the listener may be stopped for
+// graceful termination (POSIX only).
 type Server struct {
+	*http.Server
 	HTTP2      bool                   // temporary while http2 is not in std lib (TODO: remove flag when part of std lib)
-	address string                 // the actual address for net.Listen to listen on
 	tls        bool                   // whether this server is serving all HTTPS hosts or not
 	vhosts     map[string]virtualHost // virtual hosts keyed by their address
+	listener   ListenerFile           // the listener which is bound to the socket
+	listenerMu sync.Mutex             // protects listener
+	httpWg     sync.WaitGroup         // used to wait on outstanding connections
+}
+
+type ListenerFile interface {
+	net.Listener
+	File() (*os.File, error)
 }
 
 // New creates a new Server which will bind to addr and serve
@@ -36,14 +49,30 @@ func New(addr string, configs []Config) (*Server, error) {
 	}
 
 	s := &Server{
-		address: addr,
+		Server: &http.Server{
+			Addr: addr,
+			// TODO: Make these values configurable?
+			// ReadTimeout:    2 * time.Minute,
+			// WriteTimeout:   2 * time.Minute,
+			// MaxHeaderBytes: 1 << 16,
+		},
 		tls:    tls,
 		vhosts: make(map[string]virtualHost),
 	}
+	s.Handler = s // TODO: this is weird
+
+	// We have to bound our wg with one increment
+	// to prevent a "race condition" that is hard-coded
+	// into sync.WaitGroup.Wait() - basically, an add
+	// with a positive delta must be guaranteed to
+	// occur before Wait() is called on the wg.
+	fmt.Println("+1 (new)")
+	s.httpWg.Add(1)
 
+	// Set up each virtualhost
 	for _, conf := range configs {
 		if _, exists := s.vhosts[conf.Host]; exists {
-			return nil, fmt.Errorf("cannot serve %s - host already defined for address %s", conf.Address(), s.address)
+			return nil, fmt.Errorf("cannot serve %s - host already defined for address %s", conf.Address(), s.Addr)
 		}
 
 		vh := virtualHost{config: conf}
@@ -60,98 +89,92 @@ func New(addr string, configs []Config) (*Server, error) {
 	return s, nil
 }
 
-// Serve starts the server. It blocks until the server quits.
-func (s *Server) Serve() error {
-	server := &http.Server{
-		Addr:    s.address,
-		Handler: s,
+// Serve starts the server with an existing listener. It blocks until the
+// server stops.
+func (s *Server) Serve(ln ListenerFile) error {
+	err := s.setup()
+	if err != nil {
+		return err
 	}
+	return s.serve(ln)
+}
 
-	if s.HTTP2 {
-		// TODO: This call may not be necessary after HTTP/2 is merged into std lib
-		http2.ConfigureServer(server, nil)
+// ListenAndServe starts the server with a new listener. It blocks until the server stops.
+func (s *Server) ListenAndServe() error {
+	err := s.setup()
+	if err != nil {
+		return err
 	}
 
-	for _, vh := range s.vhosts {
-		// Execute startup functions now
-		for _, start := range vh.config.Startup {
-			err := start()
+	ln, err := net.Listen("tcp", s.Addr)
 	if err != nil {
 		return err
 	}
-		}
 
-		// Execute shutdown commands on exit
-		if len(vh.config.Shutdown) > 0 {
-			go func(vh virtualHost) {
-				// Wait for signal
-				interrupt := make(chan os.Signal, 1)
-				signal.Notify(interrupt, os.Interrupt, os.Kill) // TODO: syscall.SIGQUIT? (Ctrl+\, Unix-only)
-				<-interrupt
+	return s.serve(ln.(*net.TCPListener))
+}
 
-				// Run callbacks
-				exitCode := 0
-				for _, shutdownFunc := range vh.config.Shutdown {
-					err := shutdownFunc()
-					if err != nil {
-						exitCode = 1
-						log.Println(err)
-					}
-				}
-				os.Exit(exitCode) // BUG: Other shutdown goroutines might be running; use sync.WaitGroup
-			}(vh)
-		}
+// serve prepares s to listen on ln by wrapping ln in a
+// tcpKeepAliveListener (if ln is a *net.TCPListener) and
+// then in a gracefulListener, so that keep-alive is supported
+// as well as graceful shutdown/restart. It also configures
+// TLS listener on top of that if applicable.
+func (s *Server) serve(ln ListenerFile) error {
+	if tcpLn, ok := ln.(*net.TCPListener); ok {
+		ln = tcpKeepAliveListener{TCPListener: tcpLn}
 	}
 
+	s.listenerMu.Lock()
+	s.listener = newGracefulListener(ln, &s.httpWg)
+	s.listenerMu.Unlock()
+
 	if s.tls {
 		var tlsConfigs []TLSConfig
 		for _, vh := range s.vhosts {
 			tlsConfigs = append(tlsConfigs, vh.config.TLS)
 		}
-		return ListenAndServeTLSWithSNI(server, tlsConfigs)
+		return serveTLSWithSNI(s, s.listener, tlsConfigs)
 	}
-	return server.ListenAndServe()
+
+	return s.Server.Serve(s.listener)
 }
 
-// copy from net/http/transport.go
-func cloneTLSConfig(cfg *tls.Config) *tls.Config {
-	if cfg == nil {
-		return &tls.Config{}
+// setup prepares the server s to begin listening; it should be
+// called just before the listener announces itself on the network
+// and should only be called when the server is just starting up.
+func (s *Server) setup() error {
+	if s.HTTP2 {
+		// TODO: This call may not be necessary after HTTP/2 is merged into std lib
+		http2.ConfigureServer(s.Server, nil)
+	}
+
+	// Execute startup functions now
+	for _, vh := range s.vhosts {
+		for _, startupFunc := range vh.config.Startup {
+			err := startupFunc()
+			if err != nil {
+				return err
+			}
 		}
-	return &tls.Config{
-		Rand:                     cfg.Rand,
-		Time:                     cfg.Time,
-		Certificates:             cfg.Certificates,
-		NameToCertificate:        cfg.NameToCertificate,
-		GetCertificate:           cfg.GetCertificate,
-		RootCAs:                  cfg.RootCAs,
-		NextProtos:               cfg.NextProtos,
-		ServerName:               cfg.ServerName,
-		ClientAuth:               cfg.ClientAuth,
-		ClientCAs:                cfg.ClientCAs,
-		InsecureSkipVerify:       cfg.InsecureSkipVerify,
-		CipherSuites:             cfg.CipherSuites,
-		PreferServerCipherSuites: cfg.PreferServerCipherSuites,
-		SessionTicketsDisabled:   cfg.SessionTicketsDisabled,
-		SessionTicketKey:         cfg.SessionTicketKey,
-		ClientSessionCache:       cfg.ClientSessionCache,
-		MinVersion:               cfg.MinVersion,
-		MaxVersion:               cfg.MaxVersion,
-		CurvePreferences:         cfg.CurvePreferences,
 	}
+
+	return nil
 }
 
-// ListenAndServeTLSWithSNI serves TLS with Server Name Indication (SNI) support, which allows
-// multiple sites (different hostnames) to be served from the same address. This method is
-// adapted directly from the std lib's net/http ListenAndServeTLS function, which was
-// written by the Go Authors. It has been modified to support multiple certificate/key pairs.
-func ListenAndServeTLSWithSNI(srv *http.Server, tlsConfigs []TLSConfig) error {
-	addr := srv.Addr
+// serveTLSWithSNI serves TLS with Server Name Indication (SNI) support, which allows
+// multiple sites (different hostnames) to be served from the same address. It also
+// supports client authentication if srv has it enabled. It blocks until s quits.
+//
+// This method is adapted from the std lib's net/http ServeTLS function, which was written
+// by the Go Authors. It has been modified to support multiple certificate/key pairs,
+// client authentication, and our custom Server type.
+func serveTLSWithSNI(s *Server, ln net.Listener, tlsConfigs []TLSConfig) error {
+	addr := s.Server.Addr
 	if addr == "" {
 		addr = ":https"
 	}
 
-	config := cloneTLSConfig(srv.TLSConfig)
+	config := cloneTLSConfig(s.TLSConfig)
 	if config.NextProtos == nil {
 		config.NextProtos = []string{"http/1.1"}
 	}
@@ -180,45 +203,62 @@ func ListenAndServeTLSWithSNI(srv *http.Server, tlsConfigs []TLSConfig) error {
 		return err
 	}
 
-	// Create listener and we're on our way
-	conn, err := net.Listen("tcp", addr)
-	if err != nil {
-		return err
-	}
-	tlsListener := tls.NewListener(conn, config)
+	// Create TLS listener - note that we do not replace s.listener
+	// with this TLS listener; tls.listener is unexported and does
+	// not implement the File() method we need for graceful restarts
+	// on POSIX systems.
+	ln = tls.NewListener(ln, config)
 
-	return srv.Serve(tlsListener)
+	// Begin serving; block until done
+	return s.Server.Serve(ln)
 }
 
-// setupClientAuth sets up TLS client authentication only if
-// any of the TLS configs specified at least one cert file.
-func setupClientAuth(tlsConfigs []TLSConfig, config *tls.Config) error {
-	var clientAuth bool
-	for _, cfg := range tlsConfigs {
-		if len(cfg.ClientCerts) > 0 {
-			clientAuth = true
-			break
+// Stop stops the server. It blocks until the server is
+// totally stopped. On POSIX systems, it will wait for
+// connections to close (up to a max timeout of a few
+// seconds); on Windows it will close the listener
+// immediately.
+func (s *Server) Stop() error {
+	s.Server.SetKeepAlivesEnabled(false) // TODO: Does this even do anything? :P
+
+	if runtime.GOOS != "windows" {
+		// force connections to close after timeout
+		done := make(chan struct{})
+		go func() {
+			s.httpWg.Done() // decrement our initial increment used as a barrier
+			s.httpWg.Wait()
+			close(done)
+		}()
+
+		// Wait for remaining connections to finish or
+		// force them all to close after timeout
+		select {
+		case <-time.After(5 * time.Second): // TODO: configurable?
+		case <-done:
 		}
 	}
 
-	if clientAuth {
-		pool := x509.NewCertPool()
-		for _, cfg := range tlsConfigs {
-			for _, caFile := range cfg.ClientCerts {
-				caCrt, err := ioutil.ReadFile(caFile) // Anyone that gets a cert from Matt Holt can connect
+	// Close the listener now; this stops the server and
+	s.listenerMu.Lock()
+	err := s.listener.Close()
+	s.listenerMu.Unlock()
 	if err != nil {
-					return err
-				}
-				if !pool.AppendCertsFromPEM(caCrt) {
-					return fmt.Errorf("error loading client certificate '%s': no certificates were successfully parsed", caFile)
-				}
-			}
-		}
-		config.ClientCAs = pool
-		config.ClientAuth = tls.RequireAndVerifyClientCert
+		// TODO: Better logging
+		log.Println(err)
 	}
 
-	return nil
+	return err
+}
+
+// ListenerFd gets the file descriptor of the listener.
+func (s *Server) ListenerFd() uintptr {
+	s.listenerMu.Lock()
+	defer s.listenerMu.Unlock()
+	file, err := s.listener.File()
+	if err != nil {
+		return 0
+	}
+	return file.Fd()
 }
 
 // ServeHTTP is the entry point for every request to the address that s
@@ -226,6 +266,9 @@ func setupClientAuth(tlsConfigs []TLSConfig, config *tls.Config) error {
 // defined in the Host header so that the correct virtualhost
 // (configuration and middleware stack) will handle the request.
 func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	fmt.Println("Sleeping")
+	time.Sleep(5 * time.Second)
+	fmt.Println("Unblocking")
 	defer func() {
 		// In case the user doesn't enable error middleware, we still
 		// need to make sure that we stay alive up here
@@ -260,7 +303,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		}
 	} else {
 		w.WriteHeader(http.StatusNotFound)
-		fmt.Fprintf(w, "No such host at %s", s.address)
+		fmt.Fprintf(w, "No such host at %s", s.Server.Addr)
 	}
 }
 
@@ -270,3 +313,110 @@ func DefaultErrorFunc(w http.ResponseWriter, r *http.Request, status int) {
 	w.WriteHeader(status)
 	fmt.Fprintf(w, "%d %s", status, http.StatusText(status))
 }
+
+// setupClientAuth sets up TLS client authentication only if
+// any of the TLS configs specified at least one cert file.
+func setupClientAuth(tlsConfigs []TLSConfig, config *tls.Config) error {
+	var clientAuth bool
+	for _, cfg := range tlsConfigs {
+		if len(cfg.ClientCerts) > 0 {
+			clientAuth = true
+			break
+		}
+	}
+
+	if clientAuth {
+		pool := x509.NewCertPool()
+		for _, cfg := range tlsConfigs {
+			for _, caFile := range cfg.ClientCerts {
+				caCrt, err := ioutil.ReadFile(caFile) // Anyone that gets a cert from this CA can connect
+				if err != nil {
+					return err
+				}
+				if !pool.AppendCertsFromPEM(caCrt) {
+					return fmt.Errorf("error loading client certificate '%s': no certificates were successfully parsed", caFile)
+				}
+			}
+		}
+		config.ClientCAs = pool
+		config.ClientAuth = tls.RequireAndVerifyClientCert
+	}
+
+	return nil
+}
+
+// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted
+// connections. It's used by ListenAndServe and ListenAndServeTLS so
+// dead TCP connections (e.g. closing laptop mid-download) eventually
+// go away.
+//
+// Borrowed from the Go standard library.
+type tcpKeepAliveListener struct {
+	*net.TCPListener
+}
+
+// Accept accepts the connection with a keep-alive enabled.
+func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) {
+	tc, err := ln.AcceptTCP()
+	if err != nil {
+		return
+	}
+	tc.SetKeepAlive(true)
+	tc.SetKeepAlivePeriod(3 * time.Minute)
+	return tc, nil
+}
+
+// File implements ListenerFile; returns the underlying file of the listener.
+func (ln tcpKeepAliveListener) File() (*os.File, error) {
+	return ln.TCPListener.File()
+}
+
+// copied from net/http/transport.go
+func cloneTLSConfig(cfg *tls.Config) *tls.Config {
+	if cfg == nil {
+		return &tls.Config{}
+	}
+	return &tls.Config{
+		Rand:                     cfg.Rand,
+		Time:                     cfg.Time,
+		Certificates:             cfg.Certificates,
+		NameToCertificate:        cfg.NameToCertificate,
+		GetCertificate:           cfg.GetCertificate,
+		RootCAs:                  cfg.RootCAs,
+		NextProtos:               cfg.NextProtos,
+		ServerName:               cfg.ServerName,
+		ClientAuth:               cfg.ClientAuth,
+		ClientCAs:                cfg.ClientCAs,
+		InsecureSkipVerify:       cfg.InsecureSkipVerify,
+		CipherSuites:             cfg.CipherSuites,
+		PreferServerCipherSuites: cfg.PreferServerCipherSuites,
+		SessionTicketsDisabled:   cfg.SessionTicketsDisabled,
+		SessionTicketKey:         cfg.SessionTicketKey,
+		ClientSessionCache:       cfg.ClientSessionCache,
+		MinVersion:               cfg.MinVersion,
+		MaxVersion:               cfg.MaxVersion,
+		CurvePreferences:         cfg.CurvePreferences,
+	}
+}
+
+// ShutdownCallbacks executes all the shutdown callbacks
+// for all the virtualhosts in servers, and returns all the
+// errors generated during their execution. In other words,
+// an error executing one shutdown callback does not stop
+// execution of others. Only one shutdown callback is executed
+// at a time. You must protect the servers that are passed in
+// if they are shared across threads.
+func ShutdownCallbacks(servers []*Server) []error {
+	var errs []error
+	for _, s := range servers {
+		for _, vhost := range s.vhosts {
+			for _, shutdownFunc := range vhost.config.Shutdown {
+				err := shutdownFunc()
+				if err != nil {
+					errs = append(errs, err)
+				}
+			}
+		}
+	}
+	return errs
+}