Commit 6b3c2212 authored by Matthew Holt's avatar Matthew Holt

diagnostics: AppendUnique(), restructure sets, add metrics, fix bugs

parent 703cf7bf
......@@ -44,6 +44,7 @@ import (
"time"
"github.com/mholt/caddy/caddyfile"
"github.com/mholt/caddy/diagnostics"
)
// Configurable application parameters
......@@ -573,6 +574,8 @@ func ValidateAndExecuteDirectives(cdyfile Input, inst *Instance, justValidate bo
return err
}
diagnostics.Set("num_server_blocks", len(sblocks))
return executeDirectives(inst, cdyfile.Path(), stype.Directives(), sblocks, justValidate)
}
......
......@@ -152,18 +152,18 @@ func Run() {
// Begin diagnostics (these are no-ops if diagnostics disabled)
diagnostics.Set("caddy_version", appVersion)
// TODO: plugins
diagnostics.Set("num_listeners", len(instance.Servers()))
diagnostics.Set("server_type", serverType)
diagnostics.Set("os", runtime.GOOS)
diagnostics.Set("arch", runtime.GOARCH)
diagnostics.Set("cpu", struct {
NumLogical int `json:"num_logical"`
AESNI bool `json:"aes_ni"`
BrandName string `json:"brand_name"`
BrandName string `json:"brand_name,omitempty"`
NumLogical int `json:"num_logical,omitempty"`
AESNI bool `json:"aes_ni,omitempty"`
}{
BrandName: cpuid.CPU.BrandName,
NumLogical: runtime.NumCPU(),
AESNI: cpuid.CPU.AesNi(),
BrandName: cpuid.CPU.BrandName,
})
diagnostics.StartEmitting()
......
......@@ -20,6 +20,8 @@ import (
"os"
"path/filepath"
"strings"
"github.com/mholt/caddy/diagnostics"
)
// Parse parses the input just enough to group tokens, in
......@@ -369,6 +371,7 @@ func (p *parser) directive() error {
// The directive itself is appended as a relevant token
p.block.Tokens[dir] = append(p.block.Tokens[dir], p.tokens[p.cursor])
diagnostics.AppendUnique("directives", dir)
for p.Next() {
if p.Val() == "{" {
......
......@@ -24,6 +24,8 @@ import (
"strconv"
"strings"
"sync"
"github.com/mholt/caddy/diagnostics"
)
// tlsHandler is a http.Handler that will inject a value
......@@ -97,6 +99,13 @@ func (h *tlsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if checked {
r = r.WithContext(context.WithValue(r.Context(), MitmCtxKey, mitm))
if mitm {
go diagnostics.AppendUnique("mitm", "likely")
} else {
go diagnostics.AppendUnique("mitm", "unlikely")
}
} else {
go diagnostics.AppendUnique("mitm", "unknown")
}
if mitm && h.closeOnMITM {
......
......@@ -29,7 +29,6 @@ import (
"github.com/mholt/caddy/caddyfile"
"github.com/mholt/caddy/caddyhttp/staticfiles"
"github.com/mholt/caddy/caddytls"
"github.com/mholt/caddy/diagnostics"
)
const serverType = "http"
......@@ -206,8 +205,6 @@ func (h *httpContext) MakeServers() ([]caddy.Server, error) {
}
}
diagnostics.Set("num_sites", len(h.siteConfigs))
// we must map (group) each config to a bind address
groups, err := groupSiteConfigsByListenAddr(h.siteConfigs)
if err != nil {
......
......@@ -346,7 +346,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
}()
go diagnostics.AppendUniqueString("user_agent", r.Header.Get("User-Agent"))
go diagnostics.AppendUnique("user_agent", r.Header.Get("User-Agent"))
// copy the original, unchanged URL into the context
// so it can be referenced by middlewares
......
......@@ -25,6 +25,8 @@ import (
"sync"
"sync/atomic"
"time"
"github.com/mholt/caddy/diagnostics"
)
// configGroup is a type that keys configs by their hostname
......@@ -98,6 +100,23 @@ func (cg configGroup) GetConfigForClient(clientHello *tls.ClientHelloInfo) (*tls
//
// This method is safe for use as a tls.Config.GetCertificate callback.
func (cfg *Config) GetCertificate(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) {
go diagnostics.Append("client_hello", struct {
NoSNI bool `json:"no_sni,omitempty"`
CipherSuites []uint16 `json:"cipher_suites,omitempty"`
SupportedCurves []tls.CurveID `json:"curves,omitempty"`
SupportedPoints []uint8 `json:"points,omitempty"`
SignatureSchemes []tls.SignatureScheme `json:"sig_scheme,omitempty"`
ALPN []string `json:"alpn,omitempty"`
SupportedVersions []uint16 `json:"versions,omitempty"`
}{
NoSNI: clientHello.ServerName == "",
CipherSuites: clientHello.CipherSuites,
SupportedCurves: clientHello.SupportedCurves,
SupportedPoints: clientHello.SupportedPoints,
SignatureSchemes: clientHello.SignatureSchemes,
ALPN: clientHello.SupportedProtos,
SupportedVersions: clientHello.SupportedVersions,
})
cert, err := cfg.getCertDuringHandshake(strings.ToLower(clientHello.ServerName), true, true)
return &cert.Certificate, err
}
......
......@@ -113,7 +113,7 @@ func Set(key string, val interface{}) {
// Append appends value to a list named key.
// If key is new, a new list will be created.
// If key maps to a type that is not a list,
// an error is logged, and this is a no-op.
// a panic is logged, and this is a no-op.
//
// TODO: is this function needed/useful?
func Append(key string, value interface{}) {
......@@ -142,66 +142,38 @@ func Append(key string, value interface{}) {
bufferMu.Unlock()
}
// AppendUniqueString adds value to a set named key.
// AppendUnique adds value to a set namedkey.
// Set items are unordered. Values in the set
// are unique, but repeat values are counted.
// are unique, but how many times they are
// appended is counted.
//
// If key is new, a new set will be created.
// If key maps to a type that is not a string
// set, an error is logged, and this is a no-op.
func AppendUniqueString(key, value string) {
// If key is new, a new set will be created for
// values with that key. If key maps to a type
// that is not a counting set, a panic is logged,
// and this is a no-op.
func AppendUnique(key string, value interface{}) {
if !enabled {
return
}
bufferMu.Lock()
if bufferItemCount >= maxBufferItems {
bufferMu.Unlock()
return
}
bufVal, inBuffer := buffer[key]
mapVal, mapOk := bufVal.(map[string]int)
if inBuffer && !mapOk {
bufferMu.Unlock()
log.Printf("[PANIC] Diagnostics: key %s already used for non-map value", key)
return
}
if mapVal == nil {
buffer[key] = map[string]int{value: 1}
bufferItemCount++
} else if mapOk {
mapVal[value]++
}
setVal, setOk := bufVal.(countingSet)
if inBuffer && !setOk {
bufferMu.Unlock()
}
// AppendUniqueInt adds value to a set named key.
// Set items are unordered. Values in the set
// are unique, but repeat values are counted.
//
// If key is new, a new set will be created.
// If key maps to a type that is not an integer
// set, an error is logged, and this is a no-op.
func AppendUniqueInt(key string, value int) {
if !enabled {
log.Printf("[PANIC] Diagnostics: key %s already used for non-counting-set value", key)
return
}
bufferMu.Lock()
if setVal == nil {
// ensure the buffer is not too full, then add new unique value
if bufferItemCount >= maxBufferItems {
bufferMu.Unlock()
return
}
bufVal, inBuffer := buffer[key]
mapVal, mapOk := bufVal.(map[int]int)
if inBuffer && !mapOk {
bufferMu.Unlock()
log.Printf("[PANIC] Diagnostics: key %s already used for non-map value", key)
return
}
if mapVal == nil {
buffer[key] = map[int]int{value: 1}
buffer[key] = countingSet{value: 1}
bufferItemCount++
} else if mapOk {
mapVal[value]++
} else if setOk {
// unique value already exists, so just increment counter
setVal[value]++
}
bufferMu.Unlock()
}
......@@ -209,7 +181,7 @@ func AppendUniqueInt(key string, value int) {
// Increment adds 1 to a value named key.
// If it does not exist, it is created with
// a value of 1. If key maps to a type that
// is not an integer, an error is logged,
// is not an integer, a panic is logged,
// and this is a no-op.
func Increment(key string) {
incrementOrDecrement(key, true)
......
......@@ -21,13 +21,16 @@
// collection/aggregation functions. Call StartEmitting() when you are
// ready to begin sending diagnostic updates.
//
// When collecting metrics (functions like Set, Append*, or Increment),
// it may be desirable and even recommended to run invoke them in a new
// When collecting metrics (functions like Set, AppendUnique, or Increment),
// it may be desirable and even recommended to invoke them in a new
// goroutine (use the go keyword) in case there is lock contention;
// they are thread-safe (unless noted), and you may not want them to
// block the main thread of execution. However, sometimes blocking
// may be necessary too; for example, adding startup metrics to the
// buffer before the call to StartEmitting().
//
// This package is designed to be as fast and space-efficient as reasonably
// possible, so that it does not disrupt the flow of execution.
package diagnostics
import (
......@@ -122,11 +125,6 @@ func emit(final bool) error {
continue
}
// ensure we won't slam the diagnostics server
if reply.NextUpdate < 1*time.Second {
reply.NextUpdate = defaultUpdateInterval
}
// make sure we didn't send the update too soon; if so,
// just wait and try again -- this is a special case of
// error that we handle differently, as you can see
......@@ -151,6 +149,11 @@ func emit(final bool) error {
// schedule the next update using our default update
// interval because the server might be healthy later
// ensure we won't slam the diagnostics server
if reply.NextUpdate < 1*time.Second {
reply.NextUpdate = defaultUpdateInterval
}
// schedule the next update (if this wasn't the last one and
// if the remote server didn't tell us to stop sending)
if !final && !reply.Stop {
......@@ -216,6 +219,30 @@ type Payload struct {
Data map[string]interface{} `json:"data,omitempty"`
}
// countingSet implements a set that counts how many
// times a key is inserted. It marshals to JSON in a
// way such that keys are converted to values next
// to their associated counts.
type countingSet map[interface{}]int
// MarshalJSON implements the json.Marshaler interface.
// It converts the set to an array so that the values
// are JSON object values instead of keys, since keys
// are difficult to query in databases.
func (s countingSet) MarshalJSON() ([]byte, error) {
type Item struct {
Value interface{} `json:"value"`
Count int `json:"count"`
}
var list []Item
for k, v := range s {
list = append(list, Item{Value: k, Count: v})
}
return json.Marshal(list)
}
var (
// httpClient should be used for HTTP requests. It
// is configured with a timeout for reliability.
......@@ -253,7 +280,7 @@ var (
const (
// endpoint is the base URL to remote diagnostics server;
// the instance ID will be appended to it.
endpoint = "https://diagnostics-staging.caddyserver.com/update/" // TODO: make configurable, "http://localhost:8081/update/"
endpoint = "https://diagnostics-staging.caddyserver.com/update/" // TODO: make configurable, "http://localhost:8085/update/"
// defaultUpdateInterval is how long to wait before emitting
// more diagnostic data. This value is only used if the
......
......@@ -53,29 +53,59 @@ var (
// DescribePlugins returns a string describing the registered plugins.
func DescribePlugins() string {
pl := ListPlugins()
str := "Server types:\n"
for name := range serverTypes {
for _, name := range pl["server_types"] {
str += " " + name + "\n"
}
// List the loaders in registration order
str += "\nCaddyfile loaders:\n"
for _, name := range pl["caddyfile_loaders"] {
str += " " + name + "\n"
}
if len(eventHooks) > 0 {
str += "\nEvent hook plugins:\n"
for _, name := range pl["event_hooks"] {
str += " hook." + name + "\n"
}
}
str += "\nOther plugins:\n"
for _, name := range pl["others"] {
str += " " + name + "\n"
}
return str
}
// ListPlugins makes a list of the registered plugins,
// keyed by plugin type.
func ListPlugins() map[string][]string {
p := make(map[string][]string)
// server type plugins
for name := range serverTypes {
p["server_types"] = append(p["server_types"], name)
}
// caddyfile loaders in registration order
for _, loader := range caddyfileLoaders {
str += " " + loader.name + "\n"
p["caddyfile_loaders"] = append(p["caddyfile_loaders"], loader.name)
}
if defaultCaddyfileLoader.name != "" {
str += " " + defaultCaddyfileLoader.name + "\n"
p["caddyfile_loaders"] = append(p["caddyfile_loaders"], defaultCaddyfileLoader.name)
}
// event hook plugins
if len(eventHooks) > 0 {
// List the event hook plugins
str += "\nEvent hook plugins:\n"
for hookPlugin := range eventHooks {
str += " hook." + hookPlugin + "\n"
for name := range eventHooks {
p["event_hooks"] = append(p["event_hooks"], name)
}
}
// Let's alphabetize the rest of these...
// alphabetize the rest of the plugins
var others []string
for stype, stypePlugins := range plugins {
for name := range stypePlugins {
......@@ -89,12 +119,11 @@ func DescribePlugins() string {
}
sort.Strings(others)
str += "\nOther plugins:\n"
for _, name := range others {
str += " " + name + "\n"
p["others"] = append(p["others"], name)
}
return str
return p
}
// ValidDirectives returns the list of all directives that are
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment