diagnostics: AppendUnique(), restructure sets, add metrics, fix bugs

caddy.go

@@ -44,6 +44,7 @@ import (
 	"time"
 
 	"github.com/mholt/caddy/caddyfile"
+	"github.com/mholt/caddy/diagnostics"
 )
 
 // Configurable application parameters
@@ -573,6 +574,8 @@ func ValidateAndExecuteDirectives(cdyfile Input, inst *Instance, justValidate bo
 		return err
 	}
 
+	diagnostics.Set("num_server_blocks", len(sblocks))
+
 	return executeDirectives(inst, cdyfile.Path(), stype.Directives(), sblocks, justValidate)
 }
 

caddy/caddymain/run.go

@@ -152,18 +152,18 @@ func Run() {
 
 	// Begin diagnostics (these are no-ops if diagnostics disabled)
 	diagnostics.Set("caddy_version", appVersion)
-	// TODO: plugins
 	diagnostics.Set("num_listeners", len(instance.Servers()))
+	diagnostics.Set("server_type", serverType)
 	diagnostics.Set("os", runtime.GOOS)
 	diagnostics.Set("arch", runtime.GOARCH)
 	diagnostics.Set("cpu", struct {
-		NumLogical int    `json:"num_logical"`
-		AESNI      bool   `json:"aes_ni"`
-		BrandName  string `json:"brand_name"`
+		BrandName  string `json:"brand_name,omitempty"`
+		NumLogical int    `json:"num_logical,omitempty"`
+		AESNI      bool   `json:"aes_ni,omitempty"`
 	}{
+		BrandName:  cpuid.CPU.BrandName,
 		NumLogical: runtime.NumCPU(),
 		AESNI:      cpuid.CPU.AesNi(),
-		BrandName:  cpuid.CPU.BrandName,
 	})
 	diagnostics.StartEmitting()
 

caddyfile/parse.go

@@ -20,6 +20,8 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+
+	"github.com/mholt/caddy/diagnostics"
 )
 
 // Parse parses the input just enough to group tokens, in
@@ -369,6 +371,7 @@ func (p *parser) directive() error {
 
 	// The directive itself is appended as a relevant token
 	p.block.Tokens[dir] = append(p.block.Tokens[dir], p.tokens[p.cursor])
+	diagnostics.AppendUnique("directives", dir)
 
 	for p.Next() {
 		if p.Val() == "{" {

caddyhttp/httpserver/mitm.go

@@ -24,6 +24,8 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+
+	"github.com/mholt/caddy/diagnostics"
 )
 
 // tlsHandler is a http.Handler that will inject a value
@@ -97,6 +99,13 @@ func (h *tlsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 	if checked {
 		r = r.WithContext(context.WithValue(r.Context(), MitmCtxKey, mitm))
+		if mitm {
+			go diagnostics.AppendUnique("mitm", "likely")
+		} else {
+			go diagnostics.AppendUnique("mitm", "unlikely")
+		}
+	} else {
+		go diagnostics.AppendUnique("mitm", "unknown")
 	}
 
 	if mitm && h.closeOnMITM {

caddyhttp/httpserver/plugin.go

@@ -29,7 +29,6 @@ import (
 	"github.com/mholt/caddy/caddyfile"
 	"github.com/mholt/caddy/caddyhttp/staticfiles"
 	"github.com/mholt/caddy/caddytls"
-	"github.com/mholt/caddy/diagnostics"
 )
 
 const serverType = "http"
@@ -206,8 +205,6 @@ func (h *httpContext) MakeServers() ([]caddy.Server, error) {
 		}
 	}
 
-	diagnostics.Set("num_sites", len(h.siteConfigs))
-
 	// we must map (group) each config to a bind address
 	groups, err := groupSiteConfigsByListenAddr(h.siteConfigs)
 	if err != nil {

caddyhttp/httpserver/server.go

@@ -346,7 +346,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		}
 	}()
 
-	go diagnostics.AppendUniqueString("user_agent", r.Header.Get("User-Agent"))
+	go diagnostics.AppendUnique("user_agent", r.Header.Get("User-Agent"))
 
 	// copy the original, unchanged URL into the context
 	// so it can be referenced by middlewares

caddytls/handshake.go

@@ -25,6 +25,8 @@ import (
 	"sync"
 	"sync/atomic"
 	"time"
+
+	"github.com/mholt/caddy/diagnostics"
 )
 
 // configGroup is a type that keys configs by their hostname
@@ -98,6 +100,23 @@ func (cg configGroup) GetConfigForClient(clientHello *tls.ClientHelloInfo) (*tls
 //
 // This method is safe for use as a tls.Config.GetCertificate callback.
 func (cfg *Config) GetCertificate(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) {
+	go diagnostics.Append("client_hello", struct {
+		NoSNI             bool                  `json:"no_sni,omitempty"`
+		CipherSuites      []uint16              `json:"cipher_suites,omitempty"`
+		SupportedCurves   []tls.CurveID         `json:"curves,omitempty"`
+		SupportedPoints   []uint8               `json:"points,omitempty"`
+		SignatureSchemes  []tls.SignatureScheme `json:"sig_scheme,omitempty"`
+		ALPN              []string              `json:"alpn,omitempty"`
+		SupportedVersions []uint16              `json:"versions,omitempty"`
+	}{
+		NoSNI:             clientHello.ServerName == "",
+		CipherSuites:      clientHello.CipherSuites,
+		SupportedCurves:   clientHello.SupportedCurves,
+		SupportedPoints:   clientHello.SupportedPoints,
+		SignatureSchemes:  clientHello.SignatureSchemes,
+		ALPN:              clientHello.SupportedProtos,
+		SupportedVersions: clientHello.SupportedVersions,
+	})
 	cert, err := cfg.getCertDuringHandshake(strings.ToLower(clientHello.ServerName), true, true)
 	return &cert.Certificate, err
 }

diagnostics/collection.go

@@ -113,7 +113,7 @@ func Set(key string, val interface{}) {
 // Append appends value to a list named key.
 // If key is new, a new list will be created.
 // If key maps to a type that is not a list,
-// an error is logged, and this is a no-op.
+// a panic is logged, and this is a no-op.
 //
 // TODO: is this function needed/useful?
 func Append(key string, value interface{}) {
@@ -142,66 +142,38 @@ func Append(key string, value interface{}) {
 	bufferMu.Unlock()
 }
 
-// AppendUniqueString adds value to a set named key.
+// AppendUnique adds value to a set  namedkey.
 // Set items are unordered. Values in the set
-// are unique, but repeat values are counted.
+// are unique, but how many times they are
+// appended is counted.
 //
-// If key is new, a new set will be created.
-// If key maps to a type that is not a string
-// set, an error is logged, and this is a no-op.
-func AppendUniqueString(key, value string) {
+// If key is new, a new set will be created for
+// values with that key. If key maps to a type
+// that is not a counting set, a panic is logged,
+// and this is a no-op.
+func AppendUnique(key string, value interface{}) {
 	if !enabled {
 		return
 	}
 	bufferMu.Lock()
-	if bufferItemCount >= maxBufferItems {
-		bufferMu.Unlock()
-		return
-	}
 	bufVal, inBuffer := buffer[key]
-	mapVal, mapOk := bufVal.(map[string]int)
-	if inBuffer && !mapOk {
-		bufferMu.Unlock()
-		log.Printf("[PANIC] Diagnostics: key %s already used for non-map value", key)
-		return
-	}
-	if mapVal == nil {
-		buffer[key] = map[string]int{value: 1}
-		bufferItemCount++
-	} else if mapOk {
-		mapVal[value]++
-	}
+	setVal, setOk := bufVal.(countingSet)
+	if inBuffer && !setOk {
 		bufferMu.Unlock()
-}
-
-// AppendUniqueInt adds value to a set named key.
-// Set items are unordered. Values in the set
-// are unique, but repeat values are counted.
-//
-// If key is new, a new set will be created.
-// If key maps to a type that is not an integer
-// set, an error is logged, and this is a no-op.
-func AppendUniqueInt(key string, value int) {
-	if !enabled {
+		log.Printf("[PANIC] Diagnostics: key %s already used for non-counting-set value", key)
 		return
 	}
-	bufferMu.Lock()
+	if setVal == nil {
+		// ensure the buffer is not too full, then add new unique value
 		if bufferItemCount >= maxBufferItems {
 			bufferMu.Unlock()
 			return
 		}
-	bufVal, inBuffer := buffer[key]
-	mapVal, mapOk := bufVal.(map[int]int)
-	if inBuffer && !mapOk {
-		bufferMu.Unlock()
-		log.Printf("[PANIC] Diagnostics: key %s already used for non-map value", key)
-		return
-	}
-	if mapVal == nil {
-		buffer[key] = map[int]int{value: 1}
+		buffer[key] = countingSet{value: 1}
 		bufferItemCount++
-	} else if mapOk {
-		mapVal[value]++
+	} else if setOk {
+		// unique value already exists, so just increment counter
+		setVal[value]++
 	}
 	bufferMu.Unlock()
 }
@@ -209,7 +181,7 @@ func AppendUniqueInt(key string, value int) {
 // Increment adds 1 to a value named key.
 // If it does not exist, it is created with
 // a value of 1. If key maps to a type that
-// is not an integer, an error is logged,
+// is not an integer, a panic is logged,
 // and this is a no-op.
 func Increment(key string) {
 	incrementOrDecrement(key, true)

diagnostics/diagnostics.go

@@ -21,13 +21,16 @@
 // collection/aggregation functions. Call StartEmitting() when you are
 // ready to begin sending diagnostic updates.
 //
-// When collecting metrics (functions like Set, Append*, or Increment),
-// it may be desirable and even recommended to run invoke them in a new
+// When collecting metrics (functions like Set, AppendUnique, or Increment),
+// it may be desirable and even recommended to invoke them in a new
 // goroutine (use the go keyword) in case there is lock contention;
 // they are thread-safe (unless noted), and you may not want them to
 // block the main thread of execution. However, sometimes blocking
 // may be necessary too; for example, adding startup metrics to the
 // buffer before the call to StartEmitting().
+//
+// This package is designed to be as fast and space-efficient as reasonably
+// possible, so that it does not disrupt the flow of execution.
 package diagnostics
 
 import (
@@ -122,11 +125,6 @@ func emit(final bool) error {
 			continue
 		}
 
-		// ensure we won't slam the diagnostics server
-		if reply.NextUpdate < 1*time.Second {
-			reply.NextUpdate = defaultUpdateInterval
-		}
-
 		// make sure we didn't send the update too soon; if so,
 		// just wait and try again -- this is a special case of
 		// error that we handle differently, as you can see
@@ -151,6 +149,11 @@ func emit(final bool) error {
 	// schedule the next update using our default update
 	// interval because the server might be healthy later
 
+	// ensure we won't slam the diagnostics server
+	if reply.NextUpdate < 1*time.Second {
+		reply.NextUpdate = defaultUpdateInterval
+	}
+
 	// schedule the next update (if this wasn't the last one and
 	// if the remote server didn't tell us to stop sending)
 	if !final && !reply.Stop {
@@ -216,6 +219,30 @@ type Payload struct {
 	Data map[string]interface{} `json:"data,omitempty"`
 }
 
+// countingSet implements a set that counts how many
+// times a key is inserted. It marshals to JSON in a
+// way such that keys are converted to values next
+// to their associated counts.
+type countingSet map[interface{}]int
+
+// MarshalJSON implements the json.Marshaler interface.
+// It converts the set to an array so that the values
+// are JSON object values instead of keys, since keys
+// are difficult to query in databases.
+func (s countingSet) MarshalJSON() ([]byte, error) {
+	type Item struct {
+		Value interface{} `json:"value"`
+		Count int         `json:"count"`
+	}
+	var list []Item
+
+	for k, v := range s {
+		list = append(list, Item{Value: k, Count: v})
+	}
+
+	return json.Marshal(list)
+}
+
 var (
 	// httpClient should be used for HTTP requests. It
 	// is configured with a timeout for reliability.
@@ -253,7 +280,7 @@ var (
 const (
 	// endpoint is the base URL to remote diagnostics server;
 	// the instance ID will be appended to it.
-	endpoint = "https://diagnostics-staging.caddyserver.com/update/" // TODO: make configurable, "http://localhost:8081/update/"
+	endpoint = "https://diagnostics-staging.caddyserver.com/update/" // TODO: make configurable, "http://localhost:8085/update/"
 
 	// defaultUpdateInterval is how long to wait before emitting
 	// more diagnostic data. This value is only used if the

plugins.go

@@ -53,29 +53,59 @@ var (
 
 // DescribePlugins returns a string describing the registered plugins.
 func DescribePlugins() string {
+	pl := ListPlugins()
+
 	str := "Server types:\n"
-	for name := range serverTypes {
+	for _, name := range pl["server_types"] {
 		str += "  " + name + "\n"
 	}
 
-	// List the loaders in registration order
 	str += "\nCaddyfile loaders:\n"
+	for _, name := range pl["caddyfile_loaders"] {
+		str += "  " + name + "\n"
+	}
+
+	if len(eventHooks) > 0 {
+		str += "\nEvent hook plugins:\n"
+		for _, name := range pl["event_hooks"] {
+			str += "  hook." + name + "\n"
+		}
+	}
+
+	str += "\nOther plugins:\n"
+	for _, name := range pl["others"] {
+		str += "  " + name + "\n"
+	}
+
+	return str
+}
+
+// ListPlugins makes a list of the registered plugins,
+// keyed by plugin type.
+func ListPlugins() map[string][]string {
+	p := make(map[string][]string)
+
+	// server type plugins
+	for name := range serverTypes {
+		p["server_types"] = append(p["server_types"], name)
+	}
+
+	// caddyfile loaders in registration order
 	for _, loader := range caddyfileLoaders {
-		str += "  " + loader.name + "\n"
+		p["caddyfile_loaders"] = append(p["caddyfile_loaders"], loader.name)
 	}
 	if defaultCaddyfileLoader.name != "" {
-		str += "  " + defaultCaddyfileLoader.name + "\n"
+		p["caddyfile_loaders"] = append(p["caddyfile_loaders"], defaultCaddyfileLoader.name)
 	}
 
+	// event hook plugins
 	if len(eventHooks) > 0 {
-		// List the event hook plugins
-		str += "\nEvent hook plugins:\n"
-		for hookPlugin := range eventHooks {
-			str += "  hook." + hookPlugin + "\n"
+		for name := range eventHooks {
+			p["event_hooks"] = append(p["event_hooks"], name)
 		}
 	}
 
-	// Let's alphabetize the rest of these...
+	// alphabetize the rest of the plugins
 	var others []string
 	for stype, stypePlugins := range plugins {
 		for name := range stypePlugins {
@@ -89,12 +119,11 @@ func DescribePlugins() string {
 	}
 
 	sort.Strings(others)
-	str += "\nOther plugins:\n"
 	for _, name := range others {
-		str += "  " + name + "\n"
+		p["others"] = append(p["others"], name)
 	}
 
-	return str
+	return p
 }
 
 // ValidDirectives returns the list of all directives that are