Merge branch 'sentry' into 'master'

Add Sentry remote error collection

Closes https://gitlab.com/gitlab-org/gitlab-workhorse/issues/57

Vendors:
- https://github.com/getsentry/raven-go (unnamed permissive license)
- https://github.com/certifi/gocertifi (Mozilla Public License 2.0)

Discussed licenses with @JobV; they seem OK.

See merge request !63

Godeps/Godeps.json

@@ -2,14 +2,20 @@
 	"ImportPath": "gitlab.com/gitlab-org/gitlab-workhorse",
 	"GoVersion": "go1.7",
 	"GodepVersion": "v74",
-	"Packages": [
-		"./..."
-	],
 	"Deps": [
+		{
+			"ImportPath": "github.com/certifi/gocertifi",
+			"Comment": "2016.08.31",
+			"Rev": "ec89d50f00d39494f5b3ec5cf2fe75c53467a937"
+		},
 		{
 			"ImportPath": "github.com/dgrijalva/jwt-go",
 			"Comment": "v3.0.0",
 			"Rev": "d2709f9f1f31ebcda9651b03077758c1f3a0018c"
+		},
+		{
+			"ImportPath": "github.com/getsentry/raven-go",
+			"Rev": "379f8d0a68ca237cf8893a1cdfd4f574125e2c51"
 		}
 	]
 }

internal/helper/helpers.go

@@ -8,6 +8,8 @@ import (
 	"os"
 	"os/exec"
 	"syscall"
+
+	"github.com/getsentry/raven-go"
 )
 
 func Fail500(w http.ResponseWriter, err error) {
@@ -16,6 +18,7 @@ func Fail500(w http.ResponseWriter, err error) {
 }
 
 func LogError(err error) {
+	raven.CaptureError(err, nil)
 	log.Printf("error: %v", err)
 }
 

main.go

@@ -25,6 +25,8 @@ import (
 	"time"
 
 	"gitlab.com/gitlab-org/gitlab-workhorse/internal/upstream"
+
+	"github.com/getsentry/raven-go"
 )
 
 // Current version of GitLab Workhorse
@@ -89,15 +91,20 @@ func main() {
 		}()
 	}
 
-	up := upstream.NewUpstream(
-		backendURL,
-		*authSocket,
-		Version,
-		*secretPath,
-		*documentRoot,
-		*developmentMode,
-		*proxyHeadersTimeout,
-	)
+	up := wrapRaven(
+		upstream.NewUpstream(
+			backendURL,
+			*authSocket,
+			Version,
+			*secretPath,
+			*documentRoot,
+			*developmentMode,
+			*proxyHeadersTimeout,
+		))
 
 	log.Fatal(http.Serve(listener, up))
 }
+
+func wrapRaven(h http.Handler) http.Handler {
+	return http.HandlerFunc(raven.RecoveryHandler(h.ServeHTTP))
+}

vendor/github.com/certifi/gocertifi/LICENSE

+This Source Code Form is subject to the terms of the Mozilla Public License,
+v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain
+one at http://mozilla.org/MPL/2.0/.

vendor/github.com/certifi/gocertifi/README.md

+# GoCertifi: SSL Certificates for Golang
+
+This Go package contains a CA bundle that you can reference in your Go code.
+This is useful for systems that do not have CA bundles that Golang can find
+itself, or where a uniform set of CAs is valuable.
+
+This is the same CA bundle that ships with the
+[Python Requests](https://github.com/kennethreitz/requests) library, and is a
+Golang specific port of [certifi](https://github.com/kennethreitz/certifi). The
+CA bundle is derived from Mozilla's canonical set.
+
+## Usage
+
+You can use the `gocertifi` package as follows:
+
+```go
+import "github.com/certifi/gocertifi"
+
+cert_pool, err := gocertifi.CACerts()
+```
+
+You can use the returned `*x509.CertPool` as part of an HTTP transport, for example:
+
+```go
+import (
+  "net/http"
+  "crypto/tls"
+)
+
+// Setup an HTTP client with a custom transport
+transport := &http.Transport{
+	TLSClientConfig: &tls.Config{RootCAs: cert_pool},
+}
+client := &http.Client{Transport: transport}
+
+// Make an HTTP request using our custom transport
+resp, err := client.Get("https://example.com")
+```
+
+## Detailed Documentation
+
+Import as follows:
+
+```go
+import "github.com/certifi/gocertifi"
+```
+
+### Errors
+
+```go
+var ErrParseFailed = errors.New("gocertifi: error when parsing certificates")
+```
+
+### Functions
+
+```go
+func CACerts() (*x509.CertPool, error)
+```
+CACerts builds an X.509 certificate pool containing the Mozilla CA Certificate
+bundle. Returns nil on error along with an appropriate error code.

vendor/github.com/certifi/gocertifi/tasks.py

+from invoke import task
+import requests
+
+@task
+def update(ctx):
+    r = requests.get('https://mkcert.org/generate/')
+    r.raise_for_status()
+    certs = r.content
+
+    with open('certifi.go', 'rb') as f:
+        file = f.read()
+
+    file = file.split('`\n')
+    assert len(file) == 3
+    file[1] = certs
+
+    ctx.run("rm certifi.go")
+
+    with open('certifi.go', 'wb') as f:
+        f.write('`\n'.join(file))

vendor/github.com/getsentry/raven-go/.dockerignore

+.git

vendor/github.com/getsentry/raven-go/.gitignore

+*.test
+example/example
+
+docs/_build
+docs/doctrees

vendor/github.com/getsentry/raven-go/.gitmodules

+[submodule "docs/_sentryext"]
+	path = docs/_sentryext
+	url = https://github.com/getsentry/sentry-doc-support

vendor/github.com/getsentry/raven-go/.travis.yml

+sudo: false
+language: go
+go:
+  - "1.2"
+  - "1.3"
+  - "1.4"
+  - "1.5"
+  - "1.6"
+  - "1.7"
+  - tip
+
+before_install:
+  - go install -race std
+  - go get golang.org/x/tools/cmd/cover
+  - go get -v ./...
+
+script:
+  - go test -v -race ./...
+  - go test -v -cover ./...
+
+matrix:
+  allow_failures:
+    - go: tip

vendor/github.com/getsentry/raven-go/LICENSE

+Copyright (c) 2013 Apollic Software, LLC. All rights reserved.
+Copyright (c) 2015 Functional Software, Inc. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Apollic Software, LLC nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/github.com/getsentry/raven-go/README.md

+# raven [![Build Status](https://travis-ci.org/getsentry/raven-go.png?branch=master)](https://travis-ci.org/getsentry/raven-go)
+
+raven is a Go client for the [Sentry](https://github.com/getsentry/sentry)
+event/error logging system.
+
+- [**API Documentation**](https://godoc.org/github.com/getsentry/raven-go)
+- [**Usage and Examples**](https://docs.getsentry.com/hosted/clients/go/)
+
+## Installation
+
+```text
+go get github.com/getsentry/raven-go
+```

vendor/github.com/getsentry/raven-go/exception.go

+package raven
+
+import (
+	"reflect"
+	"regexp"
+)
+
+var errorMsgPattern = regexp.MustCompile(`\A(\w+): (.+)\z`)
+
+func NewException(err error, stacktrace *Stacktrace) *Exception {
+	msg := err.Error()
+	ex := &Exception{
+		Stacktrace: stacktrace,
+		Value:      msg,
+		Type:       reflect.TypeOf(err).String(),
+	}
+	if m := errorMsgPattern.FindStringSubmatch(msg); m != nil {
+		ex.Module, ex.Value = m[1], m[2]
+	}
+	return ex
+}
+
+// https://docs.getsentry.com/hosted/clientdev/interfaces/#failure-interfaces
+type Exception struct {
+	// Required
+	Value string `json:"value"`
+
+	// Optional
+	Type       string      `json:"type,omitempty"`
+	Module     string      `json:"module,omitempty"`
+	Stacktrace *Stacktrace `json:"stacktrace,omitempty"`
+}
+
+func (e *Exception) Class() string { return "exception" }
+
+func (e *Exception) Culprit() string {
+	if e.Stacktrace == nil {
+		return ""
+	}
+	return e.Stacktrace.Culprit()
+}

vendor/github.com/getsentry/raven-go/http.go

+package raven
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"runtime/debug"
+	"strings"
+)
+
+func NewHttp(req *http.Request) *Http {
+	proto := "http"
+	if req.TLS != nil || req.Header.Get("X-Forwarded-Proto") == "https" {
+		proto = "https"
+	}
+	h := &Http{
+		Method:  req.Method,
+		Cookies: req.Header.Get("Cookie"),
+		Query:   sanitizeQuery(req.URL.Query()).Encode(),
+		URL:     proto + "://" + req.Host + req.URL.Path,
+		Headers: make(map[string]string, len(req.Header)),
+	}
+	if addr, port, err := net.SplitHostPort(req.RemoteAddr); err == nil {
+		h.Env = map[string]string{"REMOTE_ADDR": addr, "REMOTE_PORT": port}
+	}
+	for k, v := range req.Header {
+		h.Headers[k] = strings.Join(v, ",")
+	}
+	return h
+}
+
+var querySecretFields = []string{"password", "passphrase", "passwd", "secret"}
+
+func sanitizeQuery(query url.Values) url.Values {
+	for _, keyword := range querySecretFields {
+		for field := range query {
+			if strings.Contains(field, keyword) {
+				query[field] = []string{"********"}
+			}
+		}
+	}
+	return query
+}
+
+// https://docs.getsentry.com/hosted/clientdev/interfaces/#context-interfaces
+type Http struct {
+	// Required
+	URL    string `json:"url"`
+	Method string `json:"method"`
+	Query  string `json:"query_string,omitempty"`
+
+	// Optional
+	Cookies string            `json:"cookies,omitempty"`
+	Headers map[string]string `json:"headers,omitempty"`
+	Env     map[string]string `json:"env,omitempty"`
+
+	// Must be either a string or map[string]string
+	Data interface{} `json:"data,omitempty"`
+}
+
+func (h *Http) Class() string { return "request" }
+
+// Recovery handler to wrap the stdlib net/http Mux.
+// Example:
+//	http.HandleFunc("/", raven.RecoveryHandler(func(w http.ResponseWriter, r *http.Request) {
+//		...
+//	}))
+func RecoveryHandler(handler func(http.ResponseWriter, *http.Request)) func(http.ResponseWriter, *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		defer func() {
+			if rval := recover(); rval != nil {
+				debug.PrintStack()
+				rvalStr := fmt.Sprint(rval)
+				packet := NewPacket(rvalStr, NewException(errors.New(rvalStr), NewStacktrace(2, 3, nil)), NewHttp(r))
+				Capture(packet, nil)
+				w.WriteHeader(http.StatusInternalServerError)
+			}
+		}()
+
+		handler(w, r)
+	}
+}

vendor/github.com/getsentry/raven-go/interfaces.go

+package raven
+
+// https://docs.getsentry.com/hosted/clientdev/interfaces/#message-interface
+type Message struct {
+	// Required
+	Message string `json:"message"`
+
+	// Optional
+	Params []interface{} `json:"params,omitempty"`
+}
+
+func (m *Message) Class() string { return "logentry" }
+
+// https://docs.getsentry.com/hosted/clientdev/interfaces/#template-interface
+type Template struct {
+	// Required
+	Filename    string `json:"filename"`
+	Lineno      int    `json:"lineno"`
+	ContextLine string `json:"context_line"`
+
+	// Optional
+	PreContext   []string `json:"pre_context,omitempty"`
+	PostContext  []string `json:"post_context,omitempty"`
+	AbsolutePath string   `json:"abs_path,omitempty"`
+}
+
+func (t *Template) Class() string { return "template" }
+
+// https://docs.getsentry.com/hosted/clientdev/interfaces/#context-interfaces
+type User struct {
+	// All fields are optional
+	ID       string `json:"id,omitempty"`
+	Username string `json:"username,omitempty"`
+	Email    string `json:"email,omitempty"`
+	IP       string `json:"ip_address,omitempty"`
+}
+
+func (h *User) Class() string { return "user" }
+
+// https://docs.getsentry.com/hosted/clientdev/interfaces/#context-interfaces
+type Query struct {
+	// Required
+	Query string `json:"query"`
+
+	// Optional
+	Engine string `json:"engine,omitempty"`
+}
+
+func (q *Query) Class() string { return "query" }

vendor/github.com/getsentry/raven-go/runtests.sh

+#!/bin/bash
+go test -race ./...
+go test -cover ./...
+go test -v ./...

vendor/github.com/getsentry/raven-go/stacktrace.go

+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+// Some code from the runtime/debug package of the Go standard library.
+
+package raven
+
+import (
+	"bytes"
+	"go/build"
+	"io/ioutil"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync"
+)
+
+// https://docs.getsentry.com/hosted/clientdev/interfaces/#failure-interfaces
+type Stacktrace struct {
+	// Required
+	Frames []*StacktraceFrame `json:"frames"`
+}
+
+func (s *Stacktrace) Class() string { return "stacktrace" }
+
+func (s *Stacktrace) Culprit() string {
+	for i := len(s.Frames) - 1; i >= 0; i-- {
+		frame := s.Frames[i]
+		if frame.InApp == true && frame.Module != "" && frame.Function != "" {
+			return frame.Module + "." + frame.Function
+		}
+	}
+	return ""
+}
+
+type StacktraceFrame struct {
+	// At least one required
+	Filename string `json:"filename,omitempty"`
+	Function string `json:"function,omitempty"`
+	Module   string `json:"module,omitempty"`
+
+	// Optional
+	Lineno       int      `json:"lineno,omitempty"`
+	Colno        int      `json:"colno,omitempty"`
+	AbsolutePath string   `json:"abs_path,omitempty"`
+	ContextLine  string   `json:"context_line,omitempty"`
+	PreContext   []string `json:"pre_context,omitempty"`
+	PostContext  []string `json:"post_context,omitempty"`
+	InApp        bool     `json:"in_app"`
+}
+
+// Intialize and populate a new stacktrace, skipping skip frames.
+//
+// context is the number of surrounding lines that should be included for context.
+// Setting context to 3 would try to get seven lines. Setting context to -1 returns
+// one line with no surrounding context, and 0 returns no context.
+//
+// appPackagePrefixes is a list of prefixes used to check whether a package should
+// be considered "in app".
+func NewStacktrace(skip int, context int, appPackagePrefixes []string) *Stacktrace {
+	var frames []*StacktraceFrame
+	for i := 1 + skip; ; i++ {
+		pc, file, line, ok := runtime.Caller(i)
+		if !ok {
+			break
+		}
+		frame := NewStacktraceFrame(pc, file, line, context, appPackagePrefixes)
+		if frame != nil {
+			frames = append(frames, frame)
+		}
+	}
+	// If there are no frames, the entire stacktrace is nil
+	if len(frames) == 0 {
+		return nil
+	}
+	// Optimize the path where there's only 1 frame
+	if len(frames) == 1 {
+		return &Stacktrace{frames}
+	}
+	// Sentry wants the frames with the oldest first, so reverse them
+	for i, j := 0, len(frames)-1; i < j; i, j = i+1, j-1 {
+		frames[i], frames[j] = frames[j], frames[i]
+	}
+	return &Stacktrace{frames}
+}
+
+// Build a single frame using data returned from runtime.Caller.
+//
+// context is the number of surrounding lines that should be included for context.
+// Setting context to 3 would try to get seven lines. Setting context to -1 returns
+// one line with no surrounding context, and 0 returns no context.
+//
+// appPackagePrefixes is a list of prefixes used to check whether a package should
+// be considered "in app".
+func NewStacktraceFrame(pc uintptr, file string, line, context int, appPackagePrefixes []string) *StacktraceFrame {
+	frame := &StacktraceFrame{AbsolutePath: file, Filename: trimPath(file), Lineno: line, InApp: false}
+	frame.Module, frame.Function = functionName(pc)
+
+	// `runtime.goexit` is effectively a placeholder that comes from
+	// runtime/asm_amd64.s and is meaningless.
+	if frame.Module == "runtime" && frame.Function == "goexit" {
+		return nil
+	}
+
+	if frame.Module == "main" {
+		frame.InApp = true
+	} else {
+		for _, prefix := range appPackagePrefixes {
+			if strings.HasPrefix(frame.Module, prefix) && !strings.Contains(frame.Module, "vendor") && !strings.Contains(frame.Module, "third_party") {
+				frame.InApp = true
+			}
+		}
+	}
+
+	if context > 0 {
+		contextLines, lineIdx := fileContext(file, line, context)
+		if len(contextLines) > 0 {
+			for i, line := range contextLines {
+				switch {
+				case i < lineIdx:
+					frame.PreContext = append(frame.PreContext, string(line))
+				case i == lineIdx:
+					frame.ContextLine = string(line)
+				default:
+					frame.PostContext = append(frame.PostContext, string(line))
+				}
+			}
+		}
+	} else if context == -1 {
+		contextLine, _ := fileContext(file, line, 0)
+		if len(contextLine) > 0 {
+			frame.ContextLine = string(contextLine[0])
+		}
+	}
+	return frame
+}
+
+// Retrieve the name of the package and function containing the PC.
+func functionName(pc uintptr) (pack string, name string) {
+	fn := runtime.FuncForPC(pc)
+	if fn == nil {
+		return
+	}
+	name = fn.Name()
+	// We get this:
+	//	runtime/debug.*T·ptrmethod
+	// and want this:
+	//  pack = runtime/debug
+	//	name = *T.ptrmethod
+	if idx := strings.LastIndex(name, "."); idx != -1 {
+		pack = name[:idx]
+		name = name[idx+1:]
+	}
+	name = strings.Replace(name, "·", ".", -1)
+	return
+}
+
+var fileCacheLock sync.Mutex
+var fileCache = make(map[string][][]byte)
+
+func fileContext(filename string, line, context int) ([][]byte, int) {
+	fileCacheLock.Lock()
+	defer fileCacheLock.Unlock()
+	lines, ok := fileCache[filename]
+	if !ok {
+		data, err := ioutil.ReadFile(filename)
+		if err != nil {
+			return nil, 0
+		}
+		lines = bytes.Split(data, []byte{'\n'})
+		fileCache[filename] = lines
+	}
+	line-- // stack trace lines are 1-indexed
+	start := line - context
+	var idx int
+	if start < 0 {
+		start = 0
+		idx = line
+	} else {
+		idx = context
+	}
+	end := line + context + 1
+	if line >= len(lines) {
+		return nil, 0
+	}
+	if end > len(lines) {
+		end = len(lines)
+	}
+	return lines[start:end], idx
+}
+
+var trimPaths []string
+
+// Try to trim the GOROOT or GOPATH prefix off of a filename
+func trimPath(filename string) string {
+	for _, prefix := range trimPaths {
+		if trimmed := strings.TrimPrefix(filename, prefix); len(trimmed) < len(filename) {
+			return trimmed
+		}
+	}
+	return filename
+}
+
+func init() {
+	// Collect all source directories, and make sure they
+	// end in a trailing "separator"
+	for _, prefix := range build.Default.SrcDirs() {
+		if prefix[len(prefix)-1] != filepath.Separator {
+			prefix += string(filepath.Separator)
+		}
+		trimPaths = append(trimPaths, prefix)
+	}
+}

vendor/github.com/getsentry/raven-go/writer.go

+package raven
+
+type Writer struct {
+	Client *Client
+	Level  Severity
+	Logger string // Logger name reported to Sentry
+}
+
+// Write formats the byte slice p into a string, and sends a message to
+// Sentry at the severity level indicated by the Writer w.
+func (w *Writer) Write(p []byte) (int, error) {
+	message := string(p)
+
+	packet := NewPacket(message, &Message{message, nil})
+	packet.Level = w.Level
+	packet.Logger = w.Logger
+	w.Client.Capture(packet, nil)
+
+	return len(p), nil
+}