Merge branch 'zj-update-gitaly-client' into 'master'

Upgrade the gitaly client

See merge request gitlab-org/gitlab-workhorse!351

gitaly_test.go

@@ -58,7 +58,7 @@ func TestGetInfoRefsProxiedToGitalySuccessfully(t *testing.T) {
 	gitalyServer, socketPath := startGitalyServer(t, codes.OK)
 	defer gitalyServer.Stop()
 
-	gitalyAddress := "unix://" + socketPath
+	gitalyAddress := "unix:" + socketPath
 
 	apiResponse := gitOkBody(t)
 	apiResponse.GitalyServer.Address = gitalyAddress
@@ -114,7 +114,7 @@ func TestGetInfoRefsProxiedToGitalyInterruptedStream(t *testing.T) {
 	gitalyServer, socketPath := startGitalyServer(t, codes.OK)
 	defer gitalyServer.Stop()
 
-	gitalyAddress := "unix://" + socketPath
+	gitalyAddress := "unix:" + socketPath
 	apiResponse.GitalyServer.Address = gitalyAddress
 
 	ts := testAuthServer(nil, 200, apiResponse)
@@ -150,7 +150,7 @@ func TestPostReceivePackProxiedToGitalySuccessfully(t *testing.T) {
 	gitalyServer, socketPath := startGitalyServer(t, codes.OK)
 	defer gitalyServer.Stop()
 
-	apiResponse.GitalyServer.Address = "unix://" + socketPath
+	apiResponse.GitalyServer.Address = "unix:" + socketPath
 	apiResponse.GitConfigOptions = []string{"git-config-hello=world"}
 	ts := testAuthServer(nil, 200, apiResponse)
 	defer ts.Close()
@@ -194,7 +194,7 @@ func TestPostReceivePackProxiedToGitalyInterrupted(t *testing.T) {
 	gitalyServer, socketPath := startGitalyServer(t, codes.OK)
 	defer gitalyServer.Stop()
 
-	apiResponse.GitalyServer.Address = "unix://" + socketPath
+	apiResponse.GitalyServer.Address = "unix:" + socketPath
 	ts := testAuthServer(nil, 200, apiResponse)
 	defer ts.Close()
 
@@ -244,7 +244,7 @@ func TestPostUploadPackProxiedToGitalySuccessfully(t *testing.T) {
 			gitalyServer, socketPath := startGitalyServer(t, tc.code)
 			defer gitalyServer.Stop()
 
-			apiResponse.GitalyServer.Address = "unix://" + socketPath
+			apiResponse.GitalyServer.Address = "unix:" + socketPath
 			ts := testAuthServer(nil, 200, apiResponse)
 			defer ts.Close()
 
@@ -293,7 +293,7 @@ func TestPostUploadPackProxiedToGitalyInterrupted(t *testing.T) {
 	gitalyServer, socketPath := startGitalyServer(t, codes.OK)
 	defer gitalyServer.Stop()
 
-	apiResponse.GitalyServer.Address = "unix://" + socketPath
+	apiResponse.GitalyServer.Address = "unix:" + socketPath
 	ts := testAuthServer(nil, 200, apiResponse)
 	defer ts.Close()
 
@@ -330,7 +330,7 @@ func TestGetDiffProxiedToGitalySuccessfully(t *testing.T) {
 	gitalyServer, socketPath := startGitalyServer(t, codes.OK)
 	defer gitalyServer.Stop()
 
-	gitalyAddress := "unix://" + socketPath
+	gitalyAddress := "unix:" + socketPath
 	repoStorage := "default"
 	rightCommit := "e395f646b1499e8e0279445fc99a0596a65fab7e"
 	leftCommit := "8a0f2ee90d940bfb0ba1e14e8214b0649056e4ab"
@@ -350,7 +350,7 @@ func TestGetPatchProxiedToGitalySuccessfully(t *testing.T) {
 	gitalyServer, socketPath := startGitalyServer(t, codes.OK)
 	defer gitalyServer.Stop()
 
-	gitalyAddress := "unix://" + socketPath
+	gitalyAddress := "unix:" + socketPath
 	repoStorage := "default"
 	rightCommit := "e395f646b1499e8e0279445fc99a0596a65fab7e"
 	leftCommit := "8a0f2ee90d940bfb0ba1e14e8214b0649056e4ab"
@@ -370,7 +370,7 @@ func TestGetBlobProxiedToGitalyInterruptedStream(t *testing.T) {
 	gitalyServer, socketPath := startGitalyServer(t, codes.OK)
 	defer gitalyServer.Stop()
 
-	gitalyAddress := "unix://" + socketPath
+	gitalyAddress := "unix:" + socketPath
 	repoStorage := "default"
 	oid := "54fcc214b94e78d7a41a9a8fe6d87a5e59500e51"
 	repoRelativePath := "foo/bar.git"
@@ -401,7 +401,7 @@ func TestGetArchiveProxiedToGitalySuccessfully(t *testing.T) {
 	gitalyServer, socketPath := startGitalyServer(t, codes.OK)
 	defer gitalyServer.Stop()
 
-	gitalyAddress := "unix://" + socketPath
+	gitalyAddress := "unix:" + socketPath
 	repoStorage := "default"
 	oid := "54fcc214b94e78d7a41a9a8fe6d87a5e59500e51"
 	repoRelativePath := "foo/bar.git"
@@ -442,7 +442,7 @@ func TestGetArchiveProxiedToGitalyInterruptedStream(t *testing.T) {
 	gitalyServer, socketPath := startGitalyServer(t, codes.OK)
 	defer gitalyServer.Stop()
 
-	gitalyAddress := "unix://" + socketPath
+	gitalyAddress := "unix:" + socketPath
 	repoStorage := "default"
 	oid := "54fcc214b94e78d7a41a9a8fe6d87a5e59500e51"
 	repoRelativePath := "foo/bar.git"
@@ -475,7 +475,7 @@ func TestGetDiffProxiedToGitalyInterruptedStream(t *testing.T) {
 	gitalyServer, socketPath := startGitalyServer(t, codes.OK)
 	defer gitalyServer.Stop()
 
-	gitalyAddress := "unix://" + socketPath
+	gitalyAddress := "unix:" + socketPath
 	repoStorage := "default"
 	rightCommit := "e395f646b1499e8e0279445fc99a0596a65fab7e"
 	leftCommit := "8a0f2ee90d940bfb0ba1e14e8214b0649056e4ab"
@@ -507,7 +507,7 @@ func TestGetPatchProxiedToGitalyInterruptedStream(t *testing.T) {
 	gitalyServer, socketPath := startGitalyServer(t, codes.OK)
 	defer gitalyServer.Stop()
 
-	gitalyAddress := "unix://" + socketPath
+	gitalyAddress := "unix:" + socketPath
 	repoStorage := "default"
 	rightCommit := "e395f646b1499e8e0279445fc99a0596a65fab7e"
 	leftCommit := "8a0f2ee90d940bfb0ba1e14e8214b0649056e4ab"
@@ -539,7 +539,7 @@ func TestGetSnapshotProxiedToGitalySuccessfully(t *testing.T) {
 	gitalyServer, socketPath := startGitalyServer(t, codes.OK)
 	defer gitalyServer.Stop()
 
-	gitalyAddress := "unix://" + socketPath
+	gitalyAddress := "unix:" + socketPath
 	expectedBody := testhelper.GitalyGetSnapshotResponseMock
 	archiveLength := len(expectedBody)
 
@@ -561,7 +561,7 @@ func TestGetSnapshotProxiedToGitalyInterruptedStream(t *testing.T) {
 	gitalyServer, socketPath := startGitalyServer(t, codes.OK)
 	defer gitalyServer.Stop()
 
-	gitalyAddress := "unix://" + socketPath
+	gitalyAddress := "unix:" + socketPath
 
 	params := buildGetSnapshotParams(gitalyAddress, buildPbRepo("default", "foo/bar.git"))
 	resp, _, err := doSendDataRequest("/api/v4/projects/:id/snapshot", "git-snapshot", params)

vendor/gitlab.com/gitlab-org/gitaly/client/address_parser.go

+package client
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+)
+
+// extractHostFromRemoteURL will convert Gitaly-style URL addresses of the form
+// scheme://host:port to the "host:port" addresses used by `grpc.Dial`
+func extractHostFromRemoteURL(rawAddress string) (hostAndPort string, err error) {
+	u, err := url.Parse(rawAddress)
+	if err != nil {
+		return "", err
+	}
+
+	if u.Path != "" {
+		return "", fmt.Errorf("remote addresses should not have a path")
+	}
+
+	if u.Host == "" {
+		return "", fmt.Errorf("remote addresses should have a host")
+	}
+
+	return u.Host, nil
+}
+
+// extractPathFromSocketURL will convert Gitaly-style URL addresses of the form
+// unix:/path/to/socket into file paths: `/path/to/socket`
+const unixPrefix = "unix:"
+
+func extractPathFromSocketURL(rawAddress string) (socketPath string, err error) {
+	if !strings.HasPrefix(rawAddress, unixPrefix) {
+		return "", fmt.Errorf("invalid socket address: %s", rawAddress)
+	}
+
+	return strings.TrimPrefix(rawAddress, unixPrefix), nil
+}

vendor/gitlab.com/gitlab-org/gitaly/client/dial.go

@@ -3,59 +3,96 @@ package client
 import (
 	"fmt"
 	"net"
-	"net/url"
-	"strings"
 	"time"
 
+	"google.golang.org/grpc/credentials"
+
+	"net/url"
+
 	"google.golang.org/grpc"
 )
 
 // DefaultDialOpts hold the default DialOptions for connection to Gitaly over UNIX-socket
-var DefaultDialOpts = []grpc.DialOption{
-	grpc.WithInsecure(),
-}
+var DefaultDialOpts = []grpc.DialOption{}
+
+type connectionType int
+
+const (
+	invalidConnection connectionType = iota
+	tcpConnection
+	tlsConnection
+	unixConnection
+)
 
 // Dial gitaly
 func Dial(rawAddress string, connOpts []grpc.DialOption) (*grpc.ClientConn, error) {
-	network, addr, err := parseAddress(rawAddress)
+	var canonicalAddress string
+	var err error
+
+	switch getConnectionType(rawAddress) {
+	case invalidConnection:
+		return nil, fmt.Errorf("invalid connection string: %s", rawAddress)
+
+	case tlsConnection:
+		canonicalAddress, err = extractHostFromRemoteURL(rawAddress) // Ensure the form: "host:port" ...
 		if err != nil {
 			return nil, err
 		}
 
-	connOpts = append(connOpts,
-		grpc.WithDialer(func(a string, timeout time.Duration) (net.Conn, error) {
-			return net.DialTimeout(network, a, timeout)
-		}))
-	conn, err := grpc.Dial(addr, connOpts...)
+		certPool, err := systemCertPool()
 		if err != nil {
 			return nil, err
 		}
 
-	return conn, nil
-}
+		creds := credentials.NewClientTLSFromCert(certPool, "")
+		connOpts = append(connOpts, grpc.WithTransportCredentials(creds))
 
-func parseAddress(rawAddress string) (network, addr string, err error) {
-	// Parsing unix:// URL's with url.Parse does not give the result we want
-	// so we do it manually.
-	for _, prefix := range []string{"unix://", "unix:"} {
-		if strings.HasPrefix(rawAddress, prefix) {
-			return "unix", strings.TrimPrefix(rawAddress, prefix), nil
-		}
+	case tcpConnection:
+		canonicalAddress, err = extractHostFromRemoteURL(rawAddress) // Ensure the form: "host:port" ...
+		if err != nil {
+			return nil, err
 		}
+		connOpts = append(connOpts, grpc.WithInsecure())
 
-	u, err := url.Parse(rawAddress)
+	case unixConnection:
+		canonicalAddress = rawAddress // This will be overriden by the custom dialer...
+		connOpts = append(
+			connOpts,
+			grpc.WithInsecure(),
+			grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
+				path, err := extractPathFromSocketURL(addr)
 				if err != nil {
-		return "", "", err
+					return nil, err
 				}
 
-	if u.Scheme != "tcp" {
-		return "", "", fmt.Errorf("unknown scheme: %q", rawAddress)
+				return net.DialTimeout("unix", path, timeout)
+			}),
+		)
+
 	}
-	if u.Host == "" {
-		return "", "", fmt.Errorf("network tcp requires host: %q", rawAddress)
+
+	conn, err := grpc.Dial(canonicalAddress, connOpts...)
+	if err != nil {
+		return nil, err
 	}
-	if u.Path != "" {
-		return "", "", fmt.Errorf("network tcp should have no path: %q", rawAddress)
+
+	return conn, nil
+}
+
+func getConnectionType(rawAddress string) connectionType {
+	u, err := url.Parse(rawAddress)
+	if err != nil {
+		return invalidConnection
+	}
+
+	switch u.Scheme {
+	case "tls":
+		return tlsConnection
+	case "unix":
+		return unixConnection
+	case "tcp":
+		return tcpConnection
+	default:
+		return invalidConnection
 	}
-	return "tcp", u.Host, nil
 }

vendor/gitlab.com/gitlab-org/gitaly/client/pool.go

+// +build !darwin
+
+package client
+
+import "crypto/x509"
+
+// systemCertPool has an override on macOS.
+func systemCertPool() (*x509.CertPool, error) { return x509.SystemCertPool() }

vendor/gitlab.com/gitlab-org/gitaly/client/pool_darwin.go

+package client
+
+import (
+	"crypto/x509"
+	"io/ioutil"
+	"os"
+	"path"
+)
+
+// systemCertPool circumvents the fact that Go on macOS does not support
+// SSL_CERT_{DIR,FILE}.
+func systemCertPool() (*x509.CertPool, error) {
+	var certPem []byte
+
+	if f := os.Getenv("SSL_CERT_FILE"); len(f) > 0 {
+		pem, err := ioutil.ReadFile(f)
+		if err != nil {
+			return nil, err
+		}
+
+		pem = append(pem, '\n')
+		certPem = append(certPem, pem...)
+	}
+
+	if d := os.Getenv("SSL_CERT_DIR"); len(d) > 0 {
+		entries, err := ioutil.ReadDir(d)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, entry := range entries {
+			if entry.IsDir() {
+				continue
+			}
+
+			pem, err := ioutil.ReadFile(path.Join(d, entry.Name()))
+			if err != nil {
+				return nil, err
+			}
+
+			pem = append(pem, '\n')
+			certPem = append(certPem, pem...)
+		}
+	}
+
+	pool, err := x509.SystemCertPool()
+	if err != nil {
+		return nil, err
+	}
+
+	pool.AppendCertsFromPEM(certPem)
+	return pool, nil
+}

vendor/gitlab.com/gitlab-org/gitaly/client/receive_pack.go

 package client
 
 import (
+	"context"
 	"io"
 
 	"gitlab.com/gitlab-org/gitaly-proto/go/gitalypb"
 	"gitlab.com/gitlab-org/gitaly/streamio"
-
-	"golang.org/x/net/context"
 	"google.golang.org/grpc"
 )
 

vendor/gitlab.com/gitlab-org/gitaly/client/upload_archive.go

 package client
 
 import (
+	"context"
 	"io"
 
 	"gitlab.com/gitlab-org/gitaly-proto/go/gitalypb"
 	"gitlab.com/gitlab-org/gitaly/streamio"
-
-	"golang.org/x/net/context"
 	"google.golang.org/grpc"
 )
 

vendor/gitlab.com/gitlab-org/gitaly/client/upload_pack.go

 package client
 
 import (
+	"context"
 	"io"
 
 	"gitlab.com/gitlab-org/gitaly-proto/go/gitalypb"
 	"gitlab.com/gitlab-org/gitaly/streamio"
-
-	"golang.org/x/net/context"
 	"google.golang.org/grpc"
 )
 

vendor/vendor.json

@@ -405,12 +405,12 @@
 			"versionExact": "v0.120.0"
 		},
 		{
-			"checksumSHA1": "uvp/juXL0uZrtdQqcgXm4+COCO8=",
+			"checksumSHA1": "aL94A408/lEOF/XC4KDHvZnwnbk=",
 			"path": "gitlab.com/gitlab-org/gitaly/client",
-			"revision": "75643882c8c5e1abb46fe29b5c4ddac3cc0dffeb",
-			"revisionTime": "2018-10-04T18:41:07Z",
-			"version": "v0.124.0",
-			"versionExact": "v0.124.0"
+			"revision": "ef786ff72690836cc3ec1214b11bd10748271c5b",
+			"revisionTime": "2019-01-14T13:41:50Z",
+			"version": "v1.13.0",
+			"versionExact": "v1.13.0"
 		},
 		{
 			"checksumSHA1": "mifcYH0qXpoPkX5KzXoM3mterWQ=",