Merge branch 'master' into script_filename-fix

README.md

@@ -70,6 +70,8 @@ Then make sure the `caddy` binary is in your PATH.
 
 To build for other platforms, use build.go with the `--goos` and `--goarch` flags.
 
+When building from source, telemetry is enabled by default. You can disable it by changing `enableTelemetry` in run.go before compiling, or use the `-disabled-metrics` flag at runtime to disable only certain metrics.
+
 
 ## Quick Start
 

caddy.go

@@ -44,6 +44,7 @@ import (
 	"time"
 
 	"github.com/mholt/caddy/caddyfile"
+	"github.com/mholt/caddy/telemetry"
 )
 
 // Configurable application parameters
@@ -122,6 +123,7 @@ type Instance struct {
 	StorageMu sync.RWMutex
 }
 
+// Instances returns the list of instances.
 func Instances() []*Instance {
 	return instances
 }
@@ -615,6 +617,8 @@ func ValidateAndExecuteDirectives(cdyfile Input, inst *Instance, justValidate bo
 		return fmt.Errorf("error inspecting server blocks: %v", err)
 	}
 
+	telemetry.Set("num_server_blocks", len(sblocks))
+
 	return executeDirectives(inst, cdyfile.Path(), stype.Directives(), sblocks, justValidate)
 }
 
@@ -869,7 +873,7 @@ func Stop() error {
 // explicitly like a common local hostname. addr must only
 // be a host or a host:port combination.
 func IsLoopback(addr string) bool {
-	host, _, err := net.SplitHostPort(addr)
+	host, _, err := net.SplitHostPort(strings.ToLower(addr))
 	if err != nil {
 		host = addr // happens if the addr is just a hostname
 	}

caddy/caddymain/run.go

@@ -15,25 +15,27 @@
 package caddymain
 
 import (
+	"bufio"
 	"errors"
 	"flag"
 	"fmt"
 	"io/ioutil"
 	"log"
 	"os"
+	"path/filepath"
 	"runtime"
 	"strconv"
 	"strings"
 
-	"gopkg.in/natefinch/lumberjack.v2"
-
-	"github.com/xenolf/lego/acmev2"
-
+	"github.com/google/uuid"
+	"github.com/klauspost/cpuid"
 	"github.com/mholt/caddy"
-	// plug in the HTTP server type
-	_ "github.com/mholt/caddy/caddyhttp"
-
 	"github.com/mholt/caddy/caddytls"
+	"github.com/mholt/caddy/telemetry"
+	"github.com/xenolf/lego/acmev2"
+	"gopkg.in/natefinch/lumberjack.v2"
+
+	_ "github.com/mholt/caddy/caddyhttp" // plug in the HTTP server type
 	// This is where other plugins get plugged in (imported)
 )
 
@@ -45,6 +47,7 @@ func init() {
 	flag.StringVar(&caddytls.DefaultCAUrl, "ca", "https://acme-v02.api.letsencrypt.org/directory", "URL to certificate authority's ACME server directory")
 	flag.BoolVar(&caddytls.DisableHTTPChallenge, "disable-http-challenge", caddytls.DisableHTTPChallenge, "Disable the ACME HTTP challenge")
 	flag.BoolVar(&caddytls.DisableTLSSNIChallenge, "disable-tls-sni-challenge", caddytls.DisableTLSSNIChallenge, "Disable the ACME TLS-SNI challenge")
+	flag.StringVar(&disabledMetrics, "disabled-metrics", "", "Comma-separated list of telemetry metrics to disable")
 	flag.StringVar(&conf, "conf", "", "Caddyfile to load (default \""+caddy.DefaultConfigFile+"\")")
 	flag.StringVar(&cpu, "cpu", "100%", "CPU cap")
 	flag.BoolVar(&plugins, "plugins", false, "List installed plugins")
@@ -87,6 +90,16 @@ func Run() {
 		})
 	}
 
+	// initialize telemetry client
+	if enableTelemetry {
+		err := initTelemetry()
+		if err != nil {
+			mustLogFatalf("[ERROR] Initializing telemetry: %v", err)
+		}
+	} else if disabledMetrics != "" {
+		mustLogFatalf("[ERROR] Cannot disable specific metrics because telemetry is disabled")
+	}
+
 	// Check for one-time actions
 	if revoke != "" {
 		err := caddytls.Revoke(revoke)
@@ -143,6 +156,26 @@ func Run() {
 	// Execute instantiation events
 	caddy.EmitEvent(caddy.InstanceStartupEvent, instance)
 
+	// Begin telemetry (these are no-ops if telemetry disabled)
+	telemetry.Set("caddy_version", appVersion)
+	telemetry.Set("num_listeners", len(instance.Servers()))
+	telemetry.Set("server_type", serverType)
+	telemetry.Set("os", runtime.GOOS)
+	telemetry.Set("arch", runtime.GOARCH)
+	telemetry.Set("cpu", struct {
+		BrandName  string `json:"brand_name,omitempty"`
+		NumLogical int    `json:"num_logical,omitempty"`
+		AESNI      bool   `json:"aes_ni,omitempty"`
+	}{
+		BrandName:  cpuid.CPU.BrandName,
+		NumLogical: runtime.NumCPU(),
+		AESNI:      cpuid.CPU.AesNi(),
+	})
+	if containerized := detectContainer(); containerized {
+		telemetry.Set("container", containerized)
+	}
+	telemetry.StartEmitting()
+
 	// Twiddle your thumbs
 	instance.Wait()
 }
@@ -266,18 +299,129 @@ func setCPU(cpu string) error {
 	return nil
 }
 
+// detectContainer attempts to determine whether the process is
+// being run inside a container. References:
+// https://tuhrig.de/how-to-know-you-are-inside-a-docker-container/
+// https://stackoverflow.com/a/20012536/1048862
+// https://gist.github.com/anantkamath/623ce7f5432680749e087cf8cfba9b69
+func detectContainer() bool {
+	if runtime.GOOS != "linux" {
+		return false
+	}
+
+	file, err := os.Open("/proc/1/cgroup")
+	if err != nil {
+		return false
+	}
+	defer file.Close()
+
+	i := 0
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		i++
+		if i > 1000 {
+			return false
+		}
+
+		line := scanner.Text()
+		parts := strings.SplitN(line, ":", 3)
+		if len(parts) < 3 {
+			continue
+		}
+
+		if strings.Contains(parts[2], "docker") ||
+			strings.Contains(parts[2], "lxc") ||
+			strings.Contains(parts[2], "moby") {
+			return true
+		}
+	}
+
+	return false
+}
+
+// initTelemetry initializes the telemetry engine.
+func initTelemetry() error {
+	uuidFilename := filepath.Join(caddy.AssetsPath(), "uuid")
+	if customUUIDFile := os.Getenv("CADDY_UUID_FILE"); customUUIDFile != "" {
+		uuidFilename = customUUIDFile
+	}
+
+	newUUID := func() uuid.UUID {
+		id := uuid.New()
+		err := ioutil.WriteFile(uuidFilename, []byte(id.String()), 0600) // human-readable as a string
+		if err != nil {
+			log.Printf("[ERROR] Persisting instance UUID: %v", err)
+		}
+		return id
+	}
+
+	var id uuid.UUID
+
+	// load UUID from storage, or create one if we don't have one
+	if uuidFile, err := os.Open(uuidFilename); os.IsNotExist(err) {
+		// no UUID exists yet; create a new one and persist it
+		id = newUUID()
+	} else if err != nil {
+		log.Printf("[ERROR] Loading persistent UUID: %v", err)
+		id = newUUID()
+	} else {
+		defer uuidFile.Close()
+		uuidBytes, err := ioutil.ReadAll(uuidFile)
+		if err != nil {
+			log.Printf("[ERROR] Reading persistent UUID: %v", err)
+			id = newUUID()
+		} else {
+			id, err = uuid.ParseBytes(uuidBytes)
+			if err != nil {
+				log.Printf("[ERROR] Parsing UUID: %v", err)
+				id = newUUID()
+			}
+		}
+	}
+
+	// parse and check the list of disabled metrics
+	var disabledMetricsSlice []string
+	if len(disabledMetrics) > 0 {
+		if len(disabledMetrics) > 1024 {
+			// mitigate disk space exhaustion at the collection endpoint
+			return fmt.Errorf("too many metrics to disable")
+		}
+		disabledMetricsSlice = strings.Split(disabledMetrics, ",")
+		for i, metric := range disabledMetricsSlice {
+			if metric == "instance_id" || metric == "timestamp" || metric == "disabled_metrics" {
+				return fmt.Errorf("instance_id, timestamp, and disabled_metrics cannot be disabled")
+			}
+			if metric == "" {
+				disabledMetricsSlice = append(disabledMetricsSlice[:i], disabledMetricsSlice[i+1:]...)
+			}
+		}
+	}
+
+	// initialize telemetry
+	telemetry.Init(id, disabledMetricsSlice)
+
+	// if any metrics were disabled, report which ones (so we know how representative the data is)
+	if len(disabledMetricsSlice) > 0 {
+		telemetry.Set("disabled_metrics", disabledMetricsSlice)
+		log.Printf("[NOTICE] The following telemetry metrics are disabled: %s", disabledMetrics)
+	}
+
+	return nil
+}
+
 const appName = "Caddy"
 
 // Flags that control program flow or startup
 var (
-	serverType string
-	conf       string
-	cpu        string
-	logfile    string
-	revoke     string
-	version    bool
-	plugins    bool
-	validate   bool
+	serverType      string
+	conf            string
+	cpu             string
+	logfile         string
+	revoke          string
+	version         bool
+	plugins         bool
+	validate        bool
+	disabledMetrics string
 )
 
 // Build information obtained with the help of -ldflags
@@ -292,3 +436,5 @@ var (
 	gitShortStat     string // git diff-index --shortstat
 	gitFilesModified string // git diff-index --name-only HEAD
 )
+
+const enableTelemetry = true

caddyfile/parse.go

@@ -20,6 +20,8 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+
+	"github.com/mholt/caddy/telemetry"
 )
 
 // Parse parses the input just enough to group tokens, in
@@ -374,6 +376,7 @@ func (p *parser) directive() error {
 
 	// The directive itself is appended as a relevant token
 	p.block.Tokens[dir] = append(p.block.Tokens[dir], p.tokens[p.cursor])
+	telemetry.AppendUnique("directives", dir)
 
 	for p.Next() {
 		if p.Val() == "{" {

caddyhttp/httpserver/mitm_test.go

@@ -32,44 +32,48 @@ func TestParseClientHello(t *testing.T) {
 			// curl 7.51.0 (x86_64-apple-darwin16.0) libcurl/7.51.0 SecureTransport zlib/1.2.8
 			inputHex: `010000a6030358a28c73a71bdfc1f09dee13fecdc58805dcce42ac44254df548f14645f7dc2c00004400ffc02cc02bc024c023c00ac009c008c030c02fc028c027c014c013c012009f009e006b0067003900330016009d009c003d003c0035002f000a00af00ae008d008c008b01000039000a00080006001700180019000b00020100000d00120010040102010501060104030203050306030005000501000000000012000000170000`,
 			expected: rawHelloInfo{
-				cipherSuites:       []uint16{255, 49196, 49195, 49188, 49187, 49162, 49161, 49160, 49200, 49199, 49192, 49191, 49172, 49171, 49170, 159, 158, 107, 103, 57, 51, 22, 157, 156, 61, 60, 53, 47, 10, 175, 174, 141, 140, 139},
-				extensions:         []uint16{10, 11, 13, 5, 18, 23},
-				compressionMethods: []byte{0},
-				curves:             []tls.CurveID{23, 24, 25},
-				points:             []uint8{0},
+				Version:            0x303,
+				CipherSuites:       []uint16{255, 49196, 49195, 49188, 49187, 49162, 49161, 49160, 49200, 49199, 49192, 49191, 49172, 49171, 49170, 159, 158, 107, 103, 57, 51, 22, 157, 156, 61, 60, 53, 47, 10, 175, 174, 141, 140, 139},
+				Extensions:         []uint16{10, 11, 13, 5, 18, 23},
+				CompressionMethods: []byte{0},
+				Curves:             []tls.CurveID{23, 24, 25},
+				Points:             []uint8{0},
 			},
 		},
 		{
 			// Chrome 56
 			inputHex: `010000c003031dae75222dae1433a5a283ddcde8ddabaefbf16d84f250eee6fdff48cdfff8a00000201a1ac02bc02fc02cc030cca9cca8cc14cc13c013c014009c009d002f0035000a010000777a7a0000ff010001000000000e000c0000096c6f63616c686f73740017000000230000000d00140012040308040401050308050501080606010201000500050100000000001200000010000e000c02683208687474702f312e3175500000000b00020100000a000a0008aaaa001d001700182a2a000100`,
 			expected: rawHelloInfo{
-				cipherSuites:       []uint16{6682, 49195, 49199, 49196, 49200, 52393, 52392, 52244, 52243, 49171, 49172, 156, 157, 47, 53, 10},
-				extensions:         []uint16{31354, 65281, 0, 23, 35, 13, 5, 18, 16, 30032, 11, 10, 10794},
-				compressionMethods: []byte{0},
-				curves:             []tls.CurveID{43690, 29, 23, 24},
-				points:             []uint8{0},
+				Version:            0x303,
+				CipherSuites:       []uint16{6682, 49195, 49199, 49196, 49200, 52393, 52392, 52244, 52243, 49171, 49172, 156, 157, 47, 53, 10},
+				Extensions:         []uint16{31354, 65281, 0, 23, 35, 13, 5, 18, 16, 30032, 11, 10, 10794},
+				CompressionMethods: []byte{0},
+				Curves:             []tls.CurveID{43690, 29, 23, 24},
+				Points:             []uint8{0},
 			},
 		},
 		{
 			// Firefox 51
 			inputHex: `010000bd030375f9022fc3a6562467f3540d68013b2d0b961979de6129e944efe0b35531323500001ec02bc02fcca9cca8c02cc030c00ac009c013c01400330039002f0035000a010000760000000e000c0000096c6f63616c686f737400170000ff01000100000a000a0008001d001700180019000b00020100002300000010000e000c02683208687474702f312e31000500050100000000ff030000000d0020001e040305030603020308040805080604010501060102010402050206020202`,
 			expected: rawHelloInfo{
-				cipherSuites:       []uint16{49195, 49199, 52393, 52392, 49196, 49200, 49162, 49161, 49171, 49172, 51, 57, 47, 53, 10},
-				extensions:         []uint16{0, 23, 65281, 10, 11, 35, 16, 5, 65283, 13},
-				compressionMethods: []byte{0},
-				curves:             []tls.CurveID{29, 23, 24, 25},
-				points:             []uint8{0},
+				Version:            0x303,
+				CipherSuites:       []uint16{49195, 49199, 52393, 52392, 49196, 49200, 49162, 49161, 49171, 49172, 51, 57, 47, 53, 10},
+				Extensions:         []uint16{0, 23, 65281, 10, 11, 35, 16, 5, 65283, 13},
+				CompressionMethods: []byte{0},
+				Curves:             []tls.CurveID{29, 23, 24, 25},
+				Points:             []uint8{0},
 			},
 		},
 		{
 			// openssl s_client (OpenSSL 0.9.8zh 14 Jan 2016)
 			inputHex: `0100012b03035d385236b8ca7b7946fa0336f164e76bf821ed90e8de26d97cc677671b6f36380000acc030c02cc028c024c014c00a00a500a300a1009f006b006a0069006800390038003700360088008700860085c032c02ec02ac026c00fc005009d003d00350084c02fc02bc027c023c013c00900a400a200a0009e00670040003f003e0033003200310030009a0099009800970045004400430042c031c02dc029c025c00ec004009c003c002f009600410007c011c007c00cc00200050004c012c008001600130010000dc00dc003000a00ff0201000055000b000403000102000a001c001a00170019001c001b0018001a0016000e000d000b000c0009000a00230000000d0020001e060106020603050105020503040104020403030103020303020102020203000f000101`,
 			expected: rawHelloInfo{
-				cipherSuites:       []uint16{49200, 49196, 49192, 49188, 49172, 49162, 165, 163, 161, 159, 107, 106, 105, 104, 57, 56, 55, 54, 136, 135, 134, 133, 49202, 49198, 49194, 49190, 49167, 49157, 157, 61, 53, 132, 49199, 49195, 49191, 49187, 49171, 49161, 164, 162, 160, 158, 103, 64, 63, 62, 51, 50, 49, 48, 154, 153, 152, 151, 69, 68, 67, 66, 49201, 49197, 49193, 49189, 49166, 49156, 156, 60, 47, 150, 65, 7, 49169, 49159, 49164, 49154, 5, 4, 49170, 49160, 22, 19, 16, 13, 49165, 49155, 10, 255},
-				extensions:         []uint16{11, 10, 35, 13, 15},
-				compressionMethods: []byte{1, 0},
-				curves:             []tls.CurveID{23, 25, 28, 27, 24, 26, 22, 14, 13, 11, 12, 9, 10},
-				points:             []uint8{0, 1, 2},
+				Version:            0x303,
+				CipherSuites:       []uint16{49200, 49196, 49192, 49188, 49172, 49162, 165, 163, 161, 159, 107, 106, 105, 104, 57, 56, 55, 54, 136, 135, 134, 133, 49202, 49198, 49194, 49190, 49167, 49157, 157, 61, 53, 132, 49199, 49195, 49191, 49187, 49171, 49161, 164, 162, 160, 158, 103, 64, 63, 62, 51, 50, 49, 48, 154, 153, 152, 151, 69, 68, 67, 66, 49201, 49197, 49193, 49189, 49166, 49156, 156, 60, 47, 150, 65, 7, 49169, 49159, 49164, 49154, 5, 4, 49170, 49160, 22, 19, 16, 13, 49165, 49155, 10, 255},
+				Extensions:         []uint16{11, 10, 35, 13, 15},
+				CompressionMethods: []byte{1, 0},
+				Curves:             []tls.CurveID{23, 25, 28, 27, 24, 26, 22, 14, 13, 11, 12, 9, 10},
+				Points:             []uint8{0, 1, 2},
 			},
 		},
 	} {
@@ -338,8 +342,8 @@ func TestHeuristicFunctionsAndHandler(t *testing.T) {
 				(isEdge && (isChrome || isFirefox || isSafari || isTor)) ||
 				(isTor && (isChrome || isFirefox || isSafari || isEdge)) {
 				t.Errorf("[%s] Test %d: Multiple fingerprinting functions matched: "+
-					"Chrome=%v Firefox=%v Safari=%v Edge=%v Tor=%v\n\tparsed hello dec: %+v\n\tparsed hello hex: %#x\n",
-					client, i, isChrome, isFirefox, isSafari, isEdge, isTor, parsed, parsed)
+					"Chrome=%v Firefox=%v Safari=%v Edge=%v Tor=%v\n\tparsed hello dec: %+v\n",
+					client, i, isChrome, isFirefox, isSafari, isEdge, isTor, parsed)
 			}
 
 			// test the handler and detection results
@@ -367,8 +371,8 @@ func TestHeuristicFunctionsAndHandler(t *testing.T) {
 			if got != want {
 				t.Errorf("[%s] Test %d: Expected MITM=%v but got %v (type assertion OK (checked)=%v)",
 					client, i, want, got, checked)
-				t.Errorf("[%s] Test %d: Looks like Chrome=%v Firefox=%v Safari=%v Edge=%v Tor=%v\n\tparsed hello dec: %+v\n\tparsed hello hex: %#x\n",
-					client, i, isChrome, isFirefox, isSafari, isEdge, isTor, parsed, parsed)
+				t.Errorf("[%s] Test %d: Looks like Chrome=%v Firefox=%v Safari=%v Edge=%v Tor=%v\n\tparsed hello dec: %+v\n",
+					client, i, isChrome, isFirefox, isSafari, isEdge, isTor, parsed)
 			}
 		}
 	}

caddyhttp/httpserver/plugin.go

@@ -30,6 +30,7 @@ import (
 	"github.com/mholt/caddy/caddyfile"
 	"github.com/mholt/caddy/caddyhttp/staticfiles"
 	"github.com/mholt/caddy/caddytls"
+	"github.com/mholt/caddy/telemetry"
 )
 
 const serverType = "http"
@@ -66,6 +67,12 @@ func init() {
 	caddy.RegisterParsingCallback(serverType, "root", hideCaddyfile)
 	caddy.RegisterParsingCallback(serverType, "tls", activateHTTPS)
 	caddytls.RegisterConfigGetter(serverType, func(c *caddy.Controller) *caddytls.Config { return GetConfig(c).TLS })
+
+	// disable the caddytls package reporting ClientHellos
+	// to telemetry, since our MITM detector does this but
+	// with more information than the standard lib provides
+	// (as of May 2018)
+	caddytls.ClientHelloTelemetry = false
 }
 
 // hideCaddyfile hides the source/origin Caddyfile if it is within the
@@ -208,6 +215,18 @@ func (h *httpContext) InspectServerBlocks(sourceFile string, serverBlocks []cadd
 // MakeServers uses the newly-created siteConfigs to
 // create and return a list of server instances.
 func (h *httpContext) MakeServers() ([]caddy.Server, error) {
+	// make a rough estimate as to whether we're in a "production
+	// environment/system" - start by assuming that most production
+	// servers will set their default CA endpoint to a public,
+	// trusted CA (obviously not a perfect hueristic)
+	var looksLikeProductionCA bool
+	for _, publicCAEndpoint := range caddytls.KnownACMECAs {
+		if strings.Contains(caddytls.DefaultCAUrl, publicCAEndpoint) {
+			looksLikeProductionCA = true
+			break
+		}
+	}
+
 	// Iterate each site configuration and make sure that:
 	// 1) TLS is disabled for explicitly-HTTP sites (necessary
 	//    when an HTTP address shares a block containing tls)
@@ -215,7 +234,22 @@ func (h *httpContext) MakeServers() ([]caddy.Server, error) {
 	//    currently, QUIC does not support ClientAuth (TODO:
 	//    revisit this when our QUIC implementation supports it)
 	// 3) if TLS ClientAuth is used, StrictHostMatching is on
+	var atLeastOneSiteLooksLikeProduction bool
 	for _, cfg := range h.siteConfigs {
+		// see if all the addresses (both sites and
+		// listeners) are loopback to help us determine
+		// if this is a "production" instance or not
+		if !atLeastOneSiteLooksLikeProduction {
+			if !caddy.IsLoopback(cfg.Addr.Host) &&
+				!caddy.IsLoopback(cfg.ListenHost) &&
+				(caddytls.QualifiesForManagedTLS(cfg) ||
+					caddytls.HostQualifies(cfg.Addr.Host)) {
+				atLeastOneSiteLooksLikeProduction = true
+			}
+		}
+
+		// make sure TLS is disabled for explicitly-HTTP sites
+		// (necessary when HTTP address shares a block containing tls)
 		if !cfg.TLS.Enabled {
 			continue
 		}
@@ -265,6 +299,18 @@ func (h *httpContext) MakeServers() ([]caddy.Server, error) {
 		servers = append(servers, s)
 	}
 
+	// NOTE: This value is only a "good guess". Quite often, development
+	// environments will use internal DNS or a local hosts file to serve
+	// real-looking domains in local development. We can't easily tell
+	// which without doing a DNS lookup, so this guess is definitely naive,
+	// and if we ever want a better guess, we will have to do DNS lookups.
+	deploymentGuess := "dev"
+	if looksLikeProductionCA && atLeastOneSiteLooksLikeProduction {
+		deploymentGuess = "prod"
+	}
+	telemetry.Set("http_deployment_guess", deploymentGuess)
+	telemetry.Set("http_num_sites", len(h.siteConfigs))
+
 	return servers, nil
 }
 

caddyhttp/httpserver/server.go

@@ -36,6 +36,7 @@ import (
 	"github.com/mholt/caddy"
 	"github.com/mholt/caddy/caddyhttp/staticfiles"
 	"github.com/mholt/caddy/caddytls"
+	"github.com/mholt/caddy/telemetry"
 )
 
 // Server is the HTTP server implementation.
@@ -348,6 +349,16 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		}
 	}()
 
+	// record the User-Agent string (with a cap on its length to mitigate attacks)
+	ua := r.Header.Get("User-Agent")
+	if len(ua) > 512 {
+		ua = ua[:512]
+	}
+	uaHash := telemetry.FastHash([]byte(ua)) // this is a normalized field
+	go telemetry.SetNested("http_user_agent", uaHash, ua)
+	go telemetry.AppendUnique("http_user_agent_count", uaHash)
+	go telemetry.Increment("http_request_count")
+
 	// copy the original, unchanged URL into the context
 	// so it can be referenced by middlewares
 	urlCopy := *r.URL

caddytls/certificates.go

@@ -26,6 +26,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/mholt/caddy/telemetry"
 	"golang.org/x/crypto/ocsp"
 )
 
@@ -165,6 +166,7 @@ func (cfg *Config) CacheManagedCertificate(domain string) (Certificate, error) {
 	if err != nil {
 		return cert, err
 	}
+	telemetry.Increment("tls_managed_cert_count")
 	return cfg.cacheCertificate(cert), nil
 }
 
@@ -179,6 +181,7 @@ func (cfg *Config) cacheUnmanagedCertificatePEMFile(certFile, keyFile string) er
 		return err
 	}
 	cfg.cacheCertificate(cert)
+	telemetry.Increment("tls_manual_cert_count")
 	return nil
 }
 
@@ -192,6 +195,7 @@ func (cfg *Config) cacheUnmanagedCertificatePEMBytes(certBytes, keyBytes []byte)
 		return err
 	}
 	cfg.cacheCertificate(cert)
+	telemetry.Increment("tls_manual_cert_count")
 	return nil
 }
 

caddytls/client.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/mholt/caddy"
+	"github.com/mholt/caddy/telemetry"
 	"github.com/xenolf/lego/acmev2"
 )
 
@@ -273,6 +274,8 @@ func (c *ACMEClient) Obtain(name string) error {
 		break
 	}
 
+	go telemetry.Increment("tls_acme_certs_obtained")
+
 	return nil
 }
 
@@ -340,6 +343,7 @@ func (c *ACMEClient) Renew(name string) error {
 	}
 
 	caddy.EmitEvent(caddy.CertRenewEvent, name)
+	go telemetry.Increment("tls_acme_certs_renewed")
 
 	return saveCertResource(c.storage, newCertMeta)
 }
@@ -366,6 +370,8 @@ func (c *ACMEClient) Revoke(name string) error {
 		return err
 	}
 
+	go telemetry.Increment("tls_acme_certs_revoked")
+
 	err = c.storage.DeleteSite(name)
 	if err != nil {
 		return errors.New("certificate revoked, but unable to delete certificate file: " + err.Error())
@@ -417,3 +423,10 @@ func (c *nameCoordinator) Has(name string) bool {
 	c.mu.RUnlock()
 	return ok
 }
+
+// KnownACMECAs is a list of ACME directory endpoints of
+// known, public, and trusted ACME-compatible certificate
+// authorities.
+var KnownACMECAs = []string{
+	"https://acme-v02.api.letsencrypt.org/directory",
+}

caddytls/config.go

@@ -23,7 +23,7 @@ import (
 	"net/url"
 	"strings"
 
-	"github.com/codahale/aesnicheck"
+	"github.com/klauspost/cpuid"
 	"github.com/mholt/caddy"
 	"github.com/xenolf/lego/acmev2"
 )
@@ -648,7 +648,7 @@ var defaultCiphersNonAESNI = []uint16{
 //
 // See https://github.com/mholt/caddy/issues/1674
 func getPreferredDefaultCiphers() []uint16 {
-	if aesnicheck.HasAESNI() {
+	if cpuid.CPU.AesNi() {
 		return defaultCiphers
 	}
 

caddytls/config_test.go

@@ -21,7 +21,7 @@ import (
 	"reflect"
 	"testing"
 
-	"github.com/codahale/aesnicheck"
+	"github.com/klauspost/cpuid"
 )
 
 func TestConvertTLSConfigProtocolVersions(t *testing.T) {
@@ -98,7 +98,7 @@ func TestConvertTLSConfigCipherSuites(t *testing.T) {
 
 func TestGetPreferredDefaultCiphers(t *testing.T) {
 	expectedCiphers := defaultCiphers
-	if !aesnicheck.HasAESNI() {
+	if !cpuid.CPU.AesNi() {
 		expectedCiphers = defaultCiphersNonAESNI
 	}
 

caddytls/crypto.go

@@ -341,7 +341,7 @@ func standaloneTLSTicketKeyRotation(c *tls.Config, ticker *time.Ticker, exitChan
 // Do not use this for cryptographic purposes.
 func fastHash(input []byte) string {
 	h := fnv.New32a()
-	h.Write([]byte(input))
+	h.Write(input)
 	return fmt.Sprintf("%x", h.Sum32())
 }
 

caddytls/handshake.go

@@ -25,6 +25,8 @@ import (
 	"sync"
 	"sync/atomic"
 	"time"
+
+	"github.com/mholt/caddy/telemetry"
 )
 
 // configGroup is a type that keys configs by their hostname
@@ -97,7 +99,27 @@ func (cg configGroup) GetConfigForClient(clientHello *tls.ClientHelloInfo) (*tls
 //
 // This method is safe for use as a tls.Config.GetCertificate callback.
 func (cfg *Config) GetCertificate(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) {
+	if ClientHelloTelemetry && len(clientHello.SupportedVersions) > 0 {
+		// If no other plugin (such as the HTTP server type) is implementing ClientHello telemetry, we do it.
+		// NOTE: The values in the Go standard lib's ClientHelloInfo aren't guaranteed to be in order.
+		info := ClientHelloInfo{
+			Version:                   clientHello.SupportedVersions[0], // report the highest
+			CipherSuites:              clientHello.CipherSuites,
+			ExtensionsUnknown:         true, // no extension info... :(
+			CompressionMethodsUnknown: true, // no compression methods... :(
+			Curves: clientHello.SupportedCurves,
+			Points: clientHello.SupportedPoints,
+			// We also have, but do not yet use: SignatureSchemes, ServerName, and SupportedProtos (ALPN)
+			// because the standard lib parses some extensions, but our MITM detector generally doesn't.
+		}
+		go telemetry.SetNested("tls_client_hello", info.Key(), info)
+	}
+
+	// get the certificate and serve it up
 	cert, err := cfg.getCertDuringHandshake(strings.ToLower(clientHello.ServerName), true, true)
+	if err == nil {
+		go telemetry.Increment("tls_handshake_count") // TODO: This is a "best guess" for now, we need something listener-level
+	}
 	return &cert.Certificate, err
 }
 
@@ -463,6 +485,42 @@ func (cfg *Config) renewDynamicCertificate(name string, currentCert Certificate)
 	return cfg.getCertDuringHandshake(name, true, false)
 }
 
+// ClientHelloInfo is our own version of the standard lib's
+// tls.ClientHelloInfo. As of May 2018, any fields populated
+// by the Go standard library are not guaranteed to have their
+// values in the original order as on the wire.
+type ClientHelloInfo struct {
+	Version            uint16        `json:"version,omitempty"`
+	CipherSuites       []uint16      `json:"cipher_suites,omitempty"`
+	Extensions         []uint16      `json:"extensions,omitempty"`
+	CompressionMethods []byte        `json:"compression,omitempty"`
+	Curves             []tls.CurveID `json:"curves,omitempty"`
+	Points             []uint8       `json:"points,omitempty"`
+
+	// Whether a couple of fields are unknown; if not, the key will encode
+	// differently to reflect that, as opposed to being known empty values.
+	// (some fields may be unknown depending on what package is being used;
+	// i.e. the Go standard lib doesn't expose some things)
+	// (very important to NOT encode these to JSON)
+	ExtensionsUnknown         bool `json:"-"`
+	CompressionMethodsUnknown bool `json:"-"`
+}
+
+// Key returns a standardized string form of the data in info,
+// useful for identifying duplicates.
+func (info ClientHelloInfo) Key() string {
+	extensions, compressionMethods := "?", "?"
+	if !info.ExtensionsUnknown {
+		extensions = fmt.Sprintf("%x", info.Extensions)
+	}
+	if !info.CompressionMethodsUnknown {
+		compressionMethods = fmt.Sprintf("%x", info.CompressionMethods)
+	}
+	return telemetry.FastHash([]byte(fmt.Sprintf("%x-%x-%s-%s-%x-%x",
+		info.Version, info.CipherSuites, extensions,
+		compressionMethods, info.Curves, info.Points)))
+}
+
 // obtainCertWaitChans is used to coordinate obtaining certs for each hostname.
 var obtainCertWaitChans = make(map[string]chan struct{})
 var obtainCertWaitChansMu sync.Mutex
@@ -477,3 +535,8 @@ var failedIssuanceMu sync.RWMutex
 // If this value is recent, do not make any on-demand certificate requests.
 var lastIssueTime time.Time
 var lastIssueTimeMu sync.Mutex
+
+// ClientHelloTelemetry determines whether to report
+// TLS ClientHellos to telemetry. Disable if doing
+// it from a different package.
+var ClientHelloTelemetry = true

caddytls/setup.go

@@ -28,6 +28,7 @@ import (
 	"strings"
 
 	"github.com/mholt/caddy"
+	"github.com/mholt/caddy/telemetry"
 )
 
 func init() {
@@ -174,9 +175,11 @@ func setupTLS(c *caddy.Controller) error {
 			case "max_certs":
 				c.Args(&maxCerts)
 				config.OnDemand = true
+				telemetry.Increment("tls_on_demand_count")
 			case "ask":
 				c.Args(&askURL)
 				config.OnDemand = true
+				telemetry.Increment("tls_on_demand_count")
 			case "dns":
 				args := c.RemainingArgs()
 				if len(args) != 1 {
@@ -283,6 +286,7 @@ func setupTLS(c *caddy.Controller) error {
 		if err != nil {
 			return fmt.Errorf("self-signed: %v", err)
 		}
+		telemetry.Increment("tls_self_signed_count")
 	}
 
 	return nil

dist/CHANGES.txt

 CHANGES
 
+0.11 (May 10, 2018)
+- Built with Go 1.10.2
+- Integrated optional telemetry client
+- proxy: Fixed file descriptor leak
+
+
 0.10.14 (April 19, 2018)
 - tls: Fix error handling bug when obtaining certificates
 

dist/README.txt

-CADDY 0.10.14
+CADDY 0.11
 
 Website
 	https://caddyserver.com

plugins.go

@@ -54,32 +54,58 @@ var (
 
 // DescribePlugins returns a string describing the registered plugins.
 func DescribePlugins() string {
+	pl := ListPlugins()
+
 	str := "Server types:\n"
-	for name := range serverTypes {
+	for _, name := range pl["server_types"] {
 		str += "  " + name + "\n"
 	}
 
-	// List the loaders in registration order
 	str += "\nCaddyfile loaders:\n"
+	for _, name := range pl["caddyfile_loaders"] {
+		str += "  " + name + "\n"
+	}
+
+	if len(pl["event_hooks"]) > 0 {
+		str += "\nEvent hook plugins:\n"
+		for _, name := range pl["event_hooks"] {
+			str += "  hook." + name + "\n"
+		}
+	}
+
+	str += "\nOther plugins:\n"
+	for _, name := range pl["others"] {
+		str += "  " + name + "\n"
+	}
+
+	return str
+}
+
+// ListPlugins makes a list of the registered plugins,
+// keyed by plugin type.
+func ListPlugins() map[string][]string {
+	p := make(map[string][]string)
+
+	// server type plugins
+	for name := range serverTypes {
+		p["server_types"] = append(p["server_types"], name)
+	}
+
+	// caddyfile loaders in registration order
 	for _, loader := range caddyfileLoaders {
-		str += "  " + loader.name + "\n"
+		p["caddyfile_loaders"] = append(p["caddyfile_loaders"], loader.name)
 	}
 	if defaultCaddyfileLoader.name != "" {
-		str += "  " + defaultCaddyfileLoader.name + "\n"
+		p["caddyfile_loaders"] = append(p["caddyfile_loaders"], defaultCaddyfileLoader.name)
 	}
 
 	// List the event hook plugins
-	hooks := ""
 	eventHooks.Range(func(k, _ interface{}) bool {
-		hooks += "  hook." + k.(string) + "\n"
+		p["event_hooks"] = append(p["event_hooks"], k.(string))
 		return true
 	})
-	if hooks != "" {
-		str += "\nEvent hook plugins:\n"
-		str += hooks
-	}
 
-	// Let's alphabetize the rest of these...
+	// alphabetize the rest of the plugins
 	var others []string
 	for stype, stypePlugins := range plugins {
 		for name := range stypePlugins {
@@ -93,12 +119,11 @@ func DescribePlugins() string {
 	}
 
 	sort.Strings(others)
-	str += "\nOther plugins:\n"
 	for _, name := range others {
-		str += "  " + name + "\n"
+		p["others"] = append(p["others"], name)
 	}
 
-	return str
+	return p
 }
 
 // ValidDirectives returns the list of all directives that are

sigtrap.go

@@ -19,6 +19,8 @@ import (
 	"os"
 	"os/signal"
 	"sync"
+
+	"github.com/mholt/caddy/telemetry"
 )
 
 // TrapSignals create signal handlers for all applicable signals for this
@@ -52,6 +54,9 @@ func trapSignalsCrossPlatform() {
 
 			log.Println("[INFO] SIGINT: Shutting down")
 
+			telemetry.AppendUnique("sigtrap", "SIGINT")
+			go telemetry.StopEmitting() // not guaranteed to finish in time; that's OK (just don't block!)
+
 			// important cleanup actions before shutdown callbacks
 			for _, f := range OnProcessExit {
 				f()

sigtrap_posix.go

@@ -21,6 +21,8 @@ import (
 	"os"
 	"os/signal"
 	"syscall"
+
+	"github.com/mholt/caddy/telemetry"
 )
 
 // trapSignalsPosix captures POSIX-only signals.
@@ -49,10 +51,15 @@ func trapSignalsPosix() {
 					log.Printf("[ERROR] SIGTERM stop: %v", err)
 					exitCode = 3
 				}
+
+				telemetry.AppendUnique("sigtrap", "SIGTERM")
+				go telemetry.StopEmitting() // won't finish in time, but that's OK - just don't block
+
 				os.Exit(exitCode)
 
 			case syscall.SIGUSR1:
 				log.Println("[INFO] SIGUSR1: Reloading")
+				go telemetry.AppendUnique("sigtrap", "SIGUSR1")
 
 				// Start with the existing Caddyfile
 				caddyfileToUse, inst, err := getCurrentCaddyfile()
@@ -92,12 +99,14 @@ func trapSignalsPosix() {
 
 			case syscall.SIGUSR2:
 				log.Println("[INFO] SIGUSR2: Upgrading")
+				go telemetry.AppendUnique("sigtrap", "SIGUSR2")
 				if err := Upgrade(); err != nil {
 					log.Printf("[ERROR] SIGUSR2: upgrading: %v", err)
 				}
 
 			case syscall.SIGHUP:
 				// ignore; this signal is sometimes sent outside of the user's control
+				go telemetry.AppendUnique("sigtrap", "SIGHUP")
 			}
 		}
 	}()

telemetry/collection.go

+// Copyright 2015 Light Code Labs, LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package telemetry
+
+import (
+	"fmt"
+	"hash/fnv"
+	"log"
+	"strings"
+
+	"github.com/google/uuid"
+)
+
+// Init initializes this package so that it may
+// be used. Do not call this function more than
+// once. Init panics if it is called more than
+// once or if the UUID value is empty. Once this
+// function is called, the rest of the package
+// may safely be used. If this function is not
+// called, the collector functions may still be
+// invoked, but they will be no-ops.
+//
+// Any metrics keys that are passed in the second
+// argument will be permanently disabled for the
+// lifetime of the process.
+func Init(instanceID uuid.UUID, disabledMetricsKeys []string) {
+	if enabled {
+		panic("already initialized")
+	}
+	if str := instanceID.String(); str == "" ||
+		str == "00000000-0000-0000-0000-000000000000" {
+		panic("empty UUID")
+	}
+	instanceUUID = instanceID
+	disabledMetricsMu.Lock()
+	for _, key := range disabledMetricsKeys {
+		disabledMetrics[strings.TrimSpace(key)] = false
+	}
+	disabledMetricsMu.Unlock()
+	enabled = true
+}
+
+// StartEmitting sends the current payload and begins the
+// transmission cycle for updates. This is the first
+// update sent, and future ones will be sent until
+// StopEmitting is called.
+//
+// This function is non-blocking (it spawns a new goroutine).
+//
+// This function panics if it was called more than once.
+// It is a no-op if this package was not initialized.
+func StartEmitting() {
+	if !enabled {
+		return
+	}
+	updateTimerMu.Lock()
+	if updateTimer != nil {
+		updateTimerMu.Unlock()
+		panic("updates already started")
+	}
+	updateTimerMu.Unlock()
+	updateMu.Lock()
+	if updating {
+		updateMu.Unlock()
+		panic("update already in progress")
+	}
+	updateMu.Unlock()
+	go logEmit(false)
+}
+
+// StopEmitting sends the current payload and terminates
+// the update cycle. No more updates will be sent.
+//
+// It is a no-op if the package was never initialized
+// or if emitting was never started.
+//
+// NOTE: This function is blocking. Run in a goroutine if
+// you want to guarantee no blocking at critical times
+// like exiting the program.
+func StopEmitting() {
+	if !enabled {
+		return
+	}
+	updateTimerMu.Lock()
+	if updateTimer == nil {
+		updateTimerMu.Unlock()
+		return
+	}
+	updateTimerMu.Unlock()
+	logEmit(true) // likely too early; may take minutes to return
+}
+
+// Reset empties the current payload buffer.
+func Reset() {
+	resetBuffer()
+}
+
+// Set puts a value in the buffer to be included
+// in the next emission. It overwrites any
+// previous value.
+//
+// This function is safe for multiple goroutines,
+// and it is recommended to call this using the
+// go keyword after the call to SendHello so it
+// doesn't block crucial code.
+func Set(key string, val interface{}) {
+	if !enabled || isDisabled(key) {
+		return
+	}
+	bufferMu.Lock()
+	if _, ok := buffer[key]; !ok {
+		if bufferItemCount >= maxBufferItems {
+			bufferMu.Unlock()
+			return
+		}
+		bufferItemCount++
+	}
+	buffer[key] = val
+	bufferMu.Unlock()
+}
+
+// SetNested puts a value in the buffer to be included
+// in the next emission, nested under the top-level key
+// as subkey. It overwrites any previous value.
+//
+// This function is safe for multiple goroutines,
+// and it is recommended to call this using the
+// go keyword after the call to SendHello so it
+// doesn't block crucial code.
+func SetNested(key, subkey string, val interface{}) {
+	if !enabled || isDisabled(key) {
+		return
+	}
+	bufferMu.Lock()
+	if topLevel, ok1 := buffer[key]; ok1 {
+		topLevelMap, ok2 := topLevel.(map[string]interface{})
+		if !ok2 {
+			bufferMu.Unlock()
+			log.Printf("[PANIC] Telemetry: key %s is already used for non-nested-map value", key)
+			return
+		}
+		if _, ok3 := topLevelMap[subkey]; !ok3 {
+			// don't exceed max buffer size
+			if bufferItemCount >= maxBufferItems {
+				bufferMu.Unlock()
+				return
+			}
+			bufferItemCount++
+		}
+		topLevelMap[subkey] = val
+	} else {
+		// don't exceed max buffer size
+		if bufferItemCount >= maxBufferItems {
+			bufferMu.Unlock()
+			return
+		}
+		bufferItemCount++
+		buffer[key] = map[string]interface{}{subkey: val}
+	}
+	bufferMu.Unlock()
+}
+
+// Append appends value to a list named key.
+// If key is new, a new list will be created.
+// If key maps to a type that is not a list,
+// a panic is logged, and this is a no-op.
+func Append(key string, value interface{}) {
+	if !enabled || isDisabled(key) {
+		return
+	}
+	bufferMu.Lock()
+	if bufferItemCount >= maxBufferItems {
+		bufferMu.Unlock()
+		return
+	}
+	// TODO: Test this...
+	bufVal, inBuffer := buffer[key]
+	sliceVal, sliceOk := bufVal.([]interface{})
+	if inBuffer && !sliceOk {
+		bufferMu.Unlock()
+		log.Printf("[PANIC] Telemetry: key %s already used for non-slice value", key)
+		return
+	}
+	if sliceVal == nil {
+		buffer[key] = []interface{}{value}
+	} else if sliceOk {
+		buffer[key] = append(sliceVal, value)
+	}
+	bufferItemCount++
+	bufferMu.Unlock()
+}
+
+// AppendUnique adds value to a set named key.
+// Set items are unordered. Values in the set
+// are unique, but how many times they are
+// appended is counted. The value must be
+// hashable.
+//
+// If key is new, a new set will be created for
+// values with that key. If key maps to a type
+// that is not a counting set, a panic is logged,
+// and this is a no-op.
+func AppendUnique(key string, value interface{}) {
+	if !enabled || isDisabled(key) {
+		return
+	}
+	bufferMu.Lock()
+	bufVal, inBuffer := buffer[key]
+	setVal, setOk := bufVal.(countingSet)
+	if inBuffer && !setOk {
+		bufferMu.Unlock()
+		log.Printf("[PANIC] Telemetry: key %s already used for non-counting-set value", key)
+		return
+	}
+	if setVal == nil {
+		// ensure the buffer is not too full, then add new unique value
+		if bufferItemCount >= maxBufferItems {
+			bufferMu.Unlock()
+			return
+		}
+		buffer[key] = countingSet{value: 1}
+		bufferItemCount++
+	} else if setOk {
+		// unique value already exists, so just increment counter
+		setVal[value]++
+	}
+	bufferMu.Unlock()
+}
+
+// Add adds amount to a value named key.
+// If it does not exist, it is created with
+// a value of 1. If key maps to a type that
+// is not an integer, a panic is logged,
+// and this is a no-op.
+func Add(key string, amount int) {
+	atomicAdd(key, amount)
+}
+
+// Increment is a shortcut for Add(key, 1)
+func Increment(key string) {
+	atomicAdd(key, 1)
+}
+
+// atomicAdd adds amount (negative to subtract)
+// to key.
+func atomicAdd(key string, amount int) {
+	if !enabled || isDisabled(key) {
+		return
+	}
+	bufferMu.Lock()
+	bufVal, inBuffer := buffer[key]
+	intVal, intOk := bufVal.(int)
+	if inBuffer && !intOk {
+		bufferMu.Unlock()
+		log.Printf("[PANIC] Telemetry: key %s already used for non-integer value", key)
+		return
+	}
+	if !inBuffer {
+		if bufferItemCount >= maxBufferItems {
+			bufferMu.Unlock()
+			return
+		}
+		bufferItemCount++
+	}
+	buffer[key] = intVal + amount
+	bufferMu.Unlock()
+}
+
+// FastHash hashes input using a 32-bit hashing algorithm
+// that is fast, and returns the hash as a hex-encoded string.
+// Do not use this for cryptographic purposes.
+func FastHash(input []byte) string {
+	h := fnv.New32a()
+	h.Write(input)
+	return fmt.Sprintf("%x", h.Sum32())
+}
+
+// isDisabled returns whether key is
+// a disabled metric key. ALL collection
+// functions should call this and not
+// save the value if this returns true.
+func isDisabled(key string) bool {
+	// for keys that are augmented with data, such as
+	// "tls_client_hello_ua:<hash>", just
+	// check the prefix "tls_client_hello_ua"
+	checkKey := key
+	if idx := strings.Index(key, ":"); idx > -1 {
+		checkKey = key[:idx]
+	}
+
+	disabledMetricsMu.RLock()
+	_, ok := disabledMetrics[checkKey]
+	disabledMetricsMu.RUnlock()
+	return ok
+}

telemetry/collection_test.go

+// Copyright 2015 Light Code Labs, LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package telemetry
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/google/uuid"
+)
+
+func TestInit(t *testing.T) {
+	reset()
+
+	id := doInit(t) // should not panic
+
+	defer func() {
+		if r := recover(); r == nil {
+			t.Errorf("Second call to Init should have panicked")
+		}
+	}()
+	Init(id, nil) // should panic
+}
+
+func TestInitEmptyUUID(t *testing.T) {
+	reset()
+	defer func() {
+		if r := recover(); r == nil {
+			t.Errorf("Call to Init with empty UUID should have panicked")
+		}
+	}()
+	Init(uuid.UUID([16]byte{}), nil)
+}
+
+func TestSet(t *testing.T) {
+	reset()
+
+	// should be no-op since we haven't called Init() yet
+	Set("test1", "foobar")
+	if _, ok := buffer["test"]; ok {
+		t.Errorf("Should not have inserted item when not initialized")
+	}
+
+	// should work after we've initialized
+	doInit(t)
+	Set("test1", "foobar")
+	val, ok := buffer["test1"]
+	if !ok {
+		t.Errorf("Expected value to be in buffer, but it wasn't")
+	} else if val.(string) != "foobar" {
+		t.Errorf("Expected 'foobar', got '%v'", val)
+	}
+
+	// should not overfill buffer
+	maxBufferItemsTmp := maxBufferItems
+	maxBufferItems = 10
+	for i := 0; i < maxBufferItems+1; i++ {
+		Set(fmt.Sprintf("overfill_%d", i), "foobar")
+	}
+	if len(buffer) > maxBufferItems {
+		t.Errorf("Should not exceed max buffer size (%d); has %d items",
+			maxBufferItems, len(buffer))
+	}
+	maxBufferItems = maxBufferItemsTmp
+
+	// Should overwrite values
+	Set("test1", "foobar2")
+	val, ok = buffer["test1"]
+	if !ok {
+		t.Errorf("Expected value to be in buffer, but it wasn't")
+	} else if val.(string) != "foobar2" {
+		t.Errorf("Expected 'foobar2', got '%v'", val)
+	}
+}
+
+// doInit calls Init() with a valid UUID
+// and returns it.
+func doInit(t *testing.T) uuid.UUID {
+	id, err := uuid.Parse(testUUID)
+	if err != nil {
+		t.Fatalf("Could not make UUID: %v", err)
+	}
+	Init(id, nil)
+	return id
+}
+
+// reset resets all the lovely package-level state;
+// can be used as a set up function in tests.
+func reset() {
+	instanceUUID = uuid.UUID{}
+	buffer = make(map[string]interface{})
+	bufferItemCount = 0
+	updating = false
+	enabled = false
+}
+
+const testUUID = "0b6cfa22-0d4c-11e8-b11b-7a0058e13201"

telemetry/telemetry_test.go

+// Copyright 2015 Light Code Labs, LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package telemetry
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+func TestMakePayloadAndResetBuffer(t *testing.T) {
+	reset()
+	id := doInit(t)
+
+	buffer = map[string]interface{}{
+		"foo1": "bar1",
+		"foo2": "bar2",
+	}
+	bufferItemCount = 2
+
+	payloadBytes, err := makePayloadAndResetBuffer()
+	if err != nil {
+		t.Fatalf("Error making payload bytes: %v", err)
+	}
+
+	if len(buffer) != 0 {
+		t.Errorf("Expected buffer len to be 0, got %d", len(buffer))
+	}
+	if bufferItemCount != 0 {
+		t.Errorf("Expected buffer item count to be 0, got %d", bufferItemCount)
+	}
+
+	var payload Payload
+	err = json.Unmarshal(payloadBytes, &payload)
+	if err != nil {
+		t.Fatalf("Error deserializing payload: %v", err)
+	}
+
+	if payload.InstanceID != id.String() {
+		t.Errorf("Expected instance ID to be set to '%s' but got '%s'", testUUID, payload.InstanceID)
+	}
+	if payload.Data == nil {
+		t.Errorf("Expected data to be set, but was nil")
+	}
+	if payload.Timestamp.IsZero() {
+		t.Errorf("Expected timestamp to be set, but was zero value")
+	}
+}

vendor/github.com/codahale/aesnicheck/asm_amd64.s

-// func HasAESNI() bool
-TEXT ·HasAESNI(SB),$0
-	XORQ AX, AX
-	INCL AX
-	CPUID
-	SHRQ $25, CX
-	ANDQ $1, CX
-	MOVB CX, ret+0(FP)
-	RET

vendor/github.com/codahale/aesnicheck/check_asm.go

-// +build amd64
-
-package aesnicheck
-
-// HasAESNI returns whether AES-NI is supported by the CPU.
-func HasAESNI() bool

vendor/github.com/codahale/aesnicheck/check_generic.go

-// +build !amd64
-
-package aesnicheck
-
-// HasAESNI returns whether AES-NI is supported by the CPU.
-func HasAESNI() bool {
-	return false
-}

vendor/github.com/codahale/aesnicheck/cmd/aesnicheck/aesnicheck.go

-// Command aesnicheck queries the CPU for AES-NI support. If AES-NI is supported,
-// aesnicheck will print "supported" and exit with a status of 0. If AES-NI is
-// not supported, aesnicheck will print "unsupported" and exit with a status of
-// -1.
-package main
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/codahale/aesnicheck"
-)
-
-func main() {
-	if aesnicheck.HasAESNI() {
-		fmt.Println("supported")
-		os.Exit(0)
-	} else {
-		fmt.Println("unsupported")
-		os.Exit(-1)
-	}
-}

vendor/github.com/codahale/aesnicheck/docs.go

-// Package aesnicheck provides a simple check to see if crypto/aes is using
-// AES-NI instructions or if the AES transform is being done in software. AES-NI
-// is constant-time, which makes it impervious to cache-level timing attacks. For
-// security-conscious deployments on public cloud infrastructure (Amazon EC2,
-// Google Compute Engine, Microsoft Azure, etc.) this may be critical.
-//
-// See http://eprint.iacr.org/2014/248 for details on cross-VM timing attacks on
-// AES keys.
-package aesnicheck

vendor/github.com/google/uuid/hash.go

@@ -27,7 +27,7 @@ var (
 func NewHash(h hash.Hash, space UUID, data []byte, version int) UUID {
 	h.Reset()
 	h.Write(space[:])
-	h.Write([]byte(data))
+	h.Write(data)
 	s := h.Sum(nil)
 	var uuid UUID
 	copy(uuid[:], s)

vendor/github.com/google/uuid/node.go

@@ -5,16 +5,14 @@
 package uuid
 
 import (
-	"net"
 	"sync"
 )
 
 var (
-	nodeMu     sync.Mutex
-	interfaces []net.Interface // cached list of interfaces
-	ifname     string          // name of interface being used
-	nodeID     [6]byte         // hardware for version 1 UUIDs
-	zeroID     [6]byte         // nodeID with only 0's
+	nodeMu sync.Mutex
+	ifname string  // name of interface being used
+	nodeID [6]byte // hardware for version 1 UUIDs
+	zeroID [6]byte // nodeID with only 0's
 )
 
 // NodeInterface returns the name of the interface from which the NodeID was
@@ -39,20 +37,12 @@ func SetNodeInterface(name string) bool {
 }
 
 func setNodeInterface(name string) bool {
-	if interfaces == nil {
-		var err error
-		interfaces, err = net.Interfaces()
-		if err != nil && name != "" {
-			return false
-		}
-	}
 
-	for _, ifs := range interfaces {
-		if len(ifs.HardwareAddr) >= 6 && (name == "" || name == ifs.Name) {
-			copy(nodeID[:], ifs.HardwareAddr)
-			ifname = ifs.Name
-			return true
-		}
+	iname, addr := getHardwareInterface(name) // null implementation for js
+	if iname != "" && addr != nil {
+		ifname = iname
+		copy(nodeID[:], addr)
+		return true
 	}
 
 	// We found no interfaces with a valid hardware address.  If name
@@ -94,9 +84,6 @@ func SetNodeID(id []byte) bool {
 // NodeID returns the 6 byte node id encoded in uuid.  It returns nil if uuid is
 // not valid.  The NodeID is only well defined for version 1 and 2 UUIDs.
 func (uuid UUID) NodeID() []byte {
-	if len(uuid) != 16 {
-		return nil
-	}
 	var node [6]byte
 	copy(node[:], uuid[10:])
 	return node[:]

vendor/github.com/google/uuid/node_js.go

+// Copyright 2017 Google Inc.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build js
+
+package uuid
+
+// getHardwareInterface returns nil values for the JS version of the code.
+// This remvoves the "net" dependency, because it is not used in the browser.
+// Using the "net" library inflates the size of the transpiled JS code by 673k bytes.
+func getHardwareInterface(name string) (string, []byte) { return "", nil }

vendor/github.com/google/uuid/node_net.go

+// Copyright 2017 Google Inc.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !js
+
+package uuid
+
+import "net"
+
+var interfaces []net.Interface // cached list of interfaces
+
+// getHardwareInterface returns the name and hardware address of interface name.
+// If name is "" then the name and hardware address of one of the system's
+// interfaces is returned.  If no interfaces are found (name does not exist or
+// there are no interfaces) then "", nil is returned.
+//
+// Only addresses of at least 6 bytes are returned.
+func getHardwareInterface(name string) (string, []byte) {
+	if interfaces == nil {
+		var err error
+		interfaces, err = net.Interfaces()
+		if err != nil {
+			return "", nil
+		}
+	}
+	for _, ifs := range interfaces {
+		if len(ifs.HardwareAddr) >= 6 && (name == "" || name == ifs.Name) {
+			return ifs.Name, ifs.HardwareAddr
+		}
+	}
+	return "", nil
+}

vendor/github.com/google/uuid/time.go

@@ -86,7 +86,7 @@ func clockSequence() int {
 	return int(clockSeq & 0x3fff)
 }
 
-// SetClockSeq sets the clock sequence to the lower 14 bits of seq.  Setting to
+// SetClockSequence sets the clock sequence to the lower 14 bits of seq.  Setting to
 // -1 causes a new sequence to be generated.
 func SetClockSequence(seq int) {
 	defer timeMu.Unlock()
@@ -100,9 +100,9 @@ func setClockSequence(seq int) {
 		randomBits(b[:]) // clock sequence
 		seq = int(b[0])<<8 | int(b[1])
 	}
-	old_seq := clockSeq
+	oldSeq := clockSeq
 	clockSeq = uint16(seq&0x3fff) | 0x8000 // Set our variant
-	if old_seq != clockSeq {
+	if oldSeq != clockSeq {
 		lasttime = 0
 	}
 }

vendor/github.com/google/uuid/uuid.go

@@ -58,11 +58,11 @@ func Parse(s string) (UUID, error) {
 		14, 16,
 		19, 21,
 		24, 26, 28, 30, 32, 34} {
-		if v, ok := xtob(s[x], s[x+1]); !ok {
+		v, ok := xtob(s[x], s[x+1])
+		if !ok {
 			return uuid, errors.New("invalid UUID format")
-		} else {
-			uuid[i] = v
 		}
+		uuid[i] = v
 	}
 	return uuid, nil
 }
@@ -88,15 +88,22 @@ func ParseBytes(b []byte) (UUID, error) {
 		14, 16,
 		19, 21,
 		24, 26, 28, 30, 32, 34} {
-		if v, ok := xtob(b[x], b[x+1]); !ok {
+		v, ok := xtob(b[x], b[x+1])
+		if !ok {
 			return uuid, errors.New("invalid UUID format")
-		} else {
-			uuid[i] = v
 		}
+		uuid[i] = v
 	}
 	return uuid, nil
 }
 
+// FromBytes creates a new UUID from a byte slice. Returns an error if the slice
+// does not have a length of 16. The bytes are copied from the slice.
+func FromBytes(b []byte) (uuid UUID, err error) {
+	err = uuid.UnmarshalBinary(b)
+	return uuid, err
+}
+
 // Must returns uuid if err is nil and panics otherwise.
 func Must(uuid UUID, err error) UUID {
 	if err != nil {

vendor/github.com/google/uuid/version4.go

@@ -14,7 +14,7 @@ func New() UUID {
 	return Must(NewRandom())
 }
 
-// NewRandom returns a Random (Version 4) UUID or panics.
+// NewRandom returns a Random (Version 4) UUID.
 //
 // The strength of the UUIDs is based on the strength of the crypto/rand
 // package.

vendor/github.com/codahale/aesnicheck/LICENSE → vendor/github.com/klauspost/cpuid/LICENSE

 The MIT License (MIT)
 
-Copyright (c) 2014 Coda Hale
+Copyright (c) 2015 Klaus Post
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -9,13 +9,14 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 copies of the Software, and to permit persons to whom the Software is
 furnished to do so, subject to the following conditions:
 
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
 
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

vendor/github.com/klauspost/cpuid/cpuid_386.s

+// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file.
+
+// +build 386,!gccgo
+
+// func asmCpuid(op uint32) (eax, ebx, ecx, edx uint32)
+TEXT ·asmCpuid(SB), 7, $0
+	XORL CX, CX
+	MOVL op+0(FP), AX
+	CPUID
+	MOVL AX, eax+4(FP)
+	MOVL BX, ebx+8(FP)
+	MOVL CX, ecx+12(FP)
+	MOVL DX, edx+16(FP)
+	RET
+
+// func asmCpuidex(op, op2 uint32) (eax, ebx, ecx, edx uint32)
+TEXT ·asmCpuidex(SB), 7, $0
+	MOVL op+0(FP), AX
+	MOVL op2+4(FP), CX
+	CPUID
+	MOVL AX, eax+8(FP)
+	MOVL BX, ebx+12(FP)
+	MOVL CX, ecx+16(FP)
+	MOVL DX, edx+20(FP)
+	RET
+
+// func xgetbv(index uint32) (eax, edx uint32)
+TEXT ·asmXgetbv(SB), 7, $0
+	MOVL index+0(FP), CX
+	BYTE $0x0f; BYTE $0x01; BYTE $0xd0 // XGETBV
+	MOVL AX, eax+4(FP)
+	MOVL DX, edx+8(FP)
+	RET
+
+// func asmRdtscpAsm() (eax, ebx, ecx, edx uint32)
+TEXT ·asmRdtscpAsm(SB), 7, $0
+	BYTE $0x0F; BYTE $0x01; BYTE $0xF9 // RDTSCP
+	MOVL AX, eax+0(FP)
+	MOVL BX, ebx+4(FP)
+	MOVL CX, ecx+8(FP)
+	MOVL DX, edx+12(FP)
+	RET

vendor/github.com/klauspost/cpuid/cpuid_amd64.s

+// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file.
+
+//+build amd64,!gccgo
+
+// func asmCpuid(op uint32) (eax, ebx, ecx, edx uint32)
+TEXT ·asmCpuid(SB), 7, $0
+	XORQ CX, CX
+	MOVL op+0(FP), AX
+	CPUID
+	MOVL AX, eax+8(FP)
+	MOVL BX, ebx+12(FP)
+	MOVL CX, ecx+16(FP)
+	MOVL DX, edx+20(FP)
+	RET
+
+// func asmCpuidex(op, op2 uint32) (eax, ebx, ecx, edx uint32)
+TEXT ·asmCpuidex(SB), 7, $0
+	MOVL op+0(FP), AX
+	MOVL op2+4(FP), CX
+	CPUID
+	MOVL AX, eax+8(FP)
+	MOVL BX, ebx+12(FP)
+	MOVL CX, ecx+16(FP)
+	MOVL DX, edx+20(FP)
+	RET
+
+// func asmXgetbv(index uint32) (eax, edx uint32)
+TEXT ·asmXgetbv(SB), 7, $0
+	MOVL index+0(FP), CX
+	BYTE $0x0f; BYTE $0x01; BYTE $0xd0 // XGETBV
+	MOVL AX, eax+8(FP)
+	MOVL DX, edx+12(FP)
+	RET
+
+// func asmRdtscpAsm() (eax, ebx, ecx, edx uint32)
+TEXT ·asmRdtscpAsm(SB), 7, $0
+	BYTE $0x0F; BYTE $0x01; BYTE $0xF9 // RDTSCP
+	MOVL AX, eax+0(FP)
+	MOVL BX, ebx+4(FP)
+	MOVL CX, ecx+8(FP)
+	MOVL DX, edx+12(FP)
+	RET

vendor/github.com/klauspost/cpuid/detect_intel.go

+// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file.
+
+// +build 386,!gccgo amd64,!gccgo
+
+package cpuid
+
+func asmCpuid(op uint32) (eax, ebx, ecx, edx uint32)
+func asmCpuidex(op, op2 uint32) (eax, ebx, ecx, edx uint32)
+func asmXgetbv(index uint32) (eax, edx uint32)
+func asmRdtscpAsm() (eax, ebx, ecx, edx uint32)
+
+func initCPU() {
+	cpuid = asmCpuid
+	cpuidex = asmCpuidex
+	xgetbv = asmXgetbv
+	rdtscpAsm = asmRdtscpAsm
+}

vendor/github.com/klauspost/cpuid/detect_ref.go

+// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file.
+
+// +build !amd64,!386 gccgo
+
+package cpuid
+
+func initCPU() {
+	cpuid = func(op uint32) (eax, ebx, ecx, edx uint32) {
+		return 0, 0, 0, 0
+	}
+
+	cpuidex = func(op, op2 uint32) (eax, ebx, ecx, edx uint32) {
+		return 0, 0, 0, 0
+	}
+
+	xgetbv = func(index uint32) (eax, edx uint32) {
+		return 0, 0
+	}
+
+	rdtscpAsm = func() (eax, ebx, ecx, edx uint32) {
+		return 0, 0, 0, 0
+	}
+}

vendor/github.com/klauspost/cpuid/generate.go

+package cpuid
+
+//go:generate go run private-gen.go
+//go:generate gofmt -w ./private

vendor/github.com/klauspost/cpuid/private/cpuid_detect_intel.go

+// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file.
+
+// +build 386,!gccgo amd64,!gccgo
+
+package cpuid
+
+func asmCpuid(op uint32) (eax, ebx, ecx, edx uint32)
+func asmCpuidex(op, op2 uint32) (eax, ebx, ecx, edx uint32)
+func asmXgetbv(index uint32) (eax, edx uint32)
+func asmRdtscpAsm() (eax, ebx, ecx, edx uint32)
+
+func initCPU() {
+	cpuid = asmCpuid
+	cpuidex = asmCpuidex
+	xgetbv = asmXgetbv
+	rdtscpAsm = asmRdtscpAsm
+}

vendor/github.com/klauspost/cpuid/private/cpuid_detect_ref.go

+// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file.
+
+// +build !amd64,!386 gccgo
+
+package cpuid
+
+func initCPU() {
+	cpuid = func(op uint32) (eax, ebx, ecx, edx uint32) {
+		return 0, 0, 0, 0
+	}
+
+	cpuidex = func(op, op2 uint32) (eax, ebx, ecx, edx uint32) {
+		return 0, 0, 0, 0
+	}
+
+	xgetbv = func(index uint32) (eax, edx uint32) {
+		return 0, 0
+	}
+
+	rdtscpAsm = func() (eax, ebx, ecx, edx uint32) {
+		return 0, 0, 0, 0
+	}
+}