Commit 254f37c7 authored by Jacob Vosmaer's avatar Jacob Vosmaer

Merge branch 'zj-update-gitaly-client' into 'master'

Upgrade the gitaly client

See merge request gitlab-org/gitlab-workhorse!351
parents 3e6f68d0 25013bfa
......@@ -58,7 +58,7 @@ func TestGetInfoRefsProxiedToGitalySuccessfully(t *testing.T) {
gitalyServer, socketPath := startGitalyServer(t, codes.OK)
defer gitalyServer.Stop()
gitalyAddress := "unix://" + socketPath
gitalyAddress := "unix:" + socketPath
apiResponse := gitOkBody(t)
apiResponse.GitalyServer.Address = gitalyAddress
......@@ -114,7 +114,7 @@ func TestGetInfoRefsProxiedToGitalyInterruptedStream(t *testing.T) {
gitalyServer, socketPath := startGitalyServer(t, codes.OK)
defer gitalyServer.Stop()
gitalyAddress := "unix://" + socketPath
gitalyAddress := "unix:" + socketPath
apiResponse.GitalyServer.Address = gitalyAddress
ts := testAuthServer(nil, 200, apiResponse)
......@@ -150,7 +150,7 @@ func TestPostReceivePackProxiedToGitalySuccessfully(t *testing.T) {
gitalyServer, socketPath := startGitalyServer(t, codes.OK)
defer gitalyServer.Stop()
apiResponse.GitalyServer.Address = "unix://" + socketPath
apiResponse.GitalyServer.Address = "unix:" + socketPath
apiResponse.GitConfigOptions = []string{"git-config-hello=world"}
ts := testAuthServer(nil, 200, apiResponse)
defer ts.Close()
......@@ -194,7 +194,7 @@ func TestPostReceivePackProxiedToGitalyInterrupted(t *testing.T) {
gitalyServer, socketPath := startGitalyServer(t, codes.OK)
defer gitalyServer.Stop()
apiResponse.GitalyServer.Address = "unix://" + socketPath
apiResponse.GitalyServer.Address = "unix:" + socketPath
ts := testAuthServer(nil, 200, apiResponse)
defer ts.Close()
......@@ -244,7 +244,7 @@ func TestPostUploadPackProxiedToGitalySuccessfully(t *testing.T) {
gitalyServer, socketPath := startGitalyServer(t, tc.code)
defer gitalyServer.Stop()
apiResponse.GitalyServer.Address = "unix://" + socketPath
apiResponse.GitalyServer.Address = "unix:" + socketPath
ts := testAuthServer(nil, 200, apiResponse)
defer ts.Close()
......@@ -293,7 +293,7 @@ func TestPostUploadPackProxiedToGitalyInterrupted(t *testing.T) {
gitalyServer, socketPath := startGitalyServer(t, codes.OK)
defer gitalyServer.Stop()
apiResponse.GitalyServer.Address = "unix://" + socketPath
apiResponse.GitalyServer.Address = "unix:" + socketPath
ts := testAuthServer(nil, 200, apiResponse)
defer ts.Close()
......@@ -330,7 +330,7 @@ func TestGetDiffProxiedToGitalySuccessfully(t *testing.T) {
gitalyServer, socketPath := startGitalyServer(t, codes.OK)
defer gitalyServer.Stop()
gitalyAddress := "unix://" + socketPath
gitalyAddress := "unix:" + socketPath
repoStorage := "default"
rightCommit := "e395f646b1499e8e0279445fc99a0596a65fab7e"
leftCommit := "8a0f2ee90d940bfb0ba1e14e8214b0649056e4ab"
......@@ -350,7 +350,7 @@ func TestGetPatchProxiedToGitalySuccessfully(t *testing.T) {
gitalyServer, socketPath := startGitalyServer(t, codes.OK)
defer gitalyServer.Stop()
gitalyAddress := "unix://" + socketPath
gitalyAddress := "unix:" + socketPath
repoStorage := "default"
rightCommit := "e395f646b1499e8e0279445fc99a0596a65fab7e"
leftCommit := "8a0f2ee90d940bfb0ba1e14e8214b0649056e4ab"
......@@ -370,7 +370,7 @@ func TestGetBlobProxiedToGitalyInterruptedStream(t *testing.T) {
gitalyServer, socketPath := startGitalyServer(t, codes.OK)
defer gitalyServer.Stop()
gitalyAddress := "unix://" + socketPath
gitalyAddress := "unix:" + socketPath
repoStorage := "default"
oid := "54fcc214b94e78d7a41a9a8fe6d87a5e59500e51"
repoRelativePath := "foo/bar.git"
......@@ -401,7 +401,7 @@ func TestGetArchiveProxiedToGitalySuccessfully(t *testing.T) {
gitalyServer, socketPath := startGitalyServer(t, codes.OK)
defer gitalyServer.Stop()
gitalyAddress := "unix://" + socketPath
gitalyAddress := "unix:" + socketPath
repoStorage := "default"
oid := "54fcc214b94e78d7a41a9a8fe6d87a5e59500e51"
repoRelativePath := "foo/bar.git"
......@@ -442,7 +442,7 @@ func TestGetArchiveProxiedToGitalyInterruptedStream(t *testing.T) {
gitalyServer, socketPath := startGitalyServer(t, codes.OK)
defer gitalyServer.Stop()
gitalyAddress := "unix://" + socketPath
gitalyAddress := "unix:" + socketPath
repoStorage := "default"
oid := "54fcc214b94e78d7a41a9a8fe6d87a5e59500e51"
repoRelativePath := "foo/bar.git"
......@@ -475,7 +475,7 @@ func TestGetDiffProxiedToGitalyInterruptedStream(t *testing.T) {
gitalyServer, socketPath := startGitalyServer(t, codes.OK)
defer gitalyServer.Stop()
gitalyAddress := "unix://" + socketPath
gitalyAddress := "unix:" + socketPath
repoStorage := "default"
rightCommit := "e395f646b1499e8e0279445fc99a0596a65fab7e"
leftCommit := "8a0f2ee90d940bfb0ba1e14e8214b0649056e4ab"
......@@ -507,7 +507,7 @@ func TestGetPatchProxiedToGitalyInterruptedStream(t *testing.T) {
gitalyServer, socketPath := startGitalyServer(t, codes.OK)
defer gitalyServer.Stop()
gitalyAddress := "unix://" + socketPath
gitalyAddress := "unix:" + socketPath
repoStorage := "default"
rightCommit := "e395f646b1499e8e0279445fc99a0596a65fab7e"
leftCommit := "8a0f2ee90d940bfb0ba1e14e8214b0649056e4ab"
......@@ -539,7 +539,7 @@ func TestGetSnapshotProxiedToGitalySuccessfully(t *testing.T) {
gitalyServer, socketPath := startGitalyServer(t, codes.OK)
defer gitalyServer.Stop()
gitalyAddress := "unix://" + socketPath
gitalyAddress := "unix:" + socketPath
expectedBody := testhelper.GitalyGetSnapshotResponseMock
archiveLength := len(expectedBody)
......@@ -561,7 +561,7 @@ func TestGetSnapshotProxiedToGitalyInterruptedStream(t *testing.T) {
gitalyServer, socketPath := startGitalyServer(t, codes.OK)
defer gitalyServer.Stop()
gitalyAddress := "unix://" + socketPath
gitalyAddress := "unix:" + socketPath
params := buildGetSnapshotParams(gitalyAddress, buildPbRepo("default", "foo/bar.git"))
resp, _, err := doSendDataRequest("/api/v4/projects/:id/snapshot", "git-snapshot", params)
......
This diff is collapsed.
package client
import (
"fmt"
"net/url"
"strings"
)
// extractHostFromRemoteURL will convert Gitaly-style URL addresses of the form
// scheme://host:port to the "host:port" addresses used by `grpc.Dial`
func extractHostFromRemoteURL(rawAddress string) (hostAndPort string, err error) {
u, err := url.Parse(rawAddress)
if err != nil {
return "", err
}
if u.Path != "" {
return "", fmt.Errorf("remote addresses should not have a path")
}
if u.Host == "" {
return "", fmt.Errorf("remote addresses should have a host")
}
return u.Host, nil
}
// extractPathFromSocketURL will convert Gitaly-style URL addresses of the form
// unix:/path/to/socket into file paths: `/path/to/socket`
const unixPrefix = "unix:"
func extractPathFromSocketURL(rawAddress string) (socketPath string, err error) {
if !strings.HasPrefix(rawAddress, unixPrefix) {
return "", fmt.Errorf("invalid socket address: %s", rawAddress)
}
return strings.TrimPrefix(rawAddress, unixPrefix), nil
}
......@@ -3,30 +3,75 @@ package client
import (
"fmt"
"net"
"net/url"
"strings"
"time"
"google.golang.org/grpc/credentials"
"net/url"
"google.golang.org/grpc"
)
// DefaultDialOpts hold the default DialOptions for connection to Gitaly over UNIX-socket
var DefaultDialOpts = []grpc.DialOption{
grpc.WithInsecure(),
}
var DefaultDialOpts = []grpc.DialOption{}
type connectionType int
const (
invalidConnection connectionType = iota
tcpConnection
tlsConnection
unixConnection
)
// Dial gitaly
func Dial(rawAddress string, connOpts []grpc.DialOption) (*grpc.ClientConn, error) {
network, addr, err := parseAddress(rawAddress)
if err != nil {
return nil, err
var canonicalAddress string
var err error
switch getConnectionType(rawAddress) {
case invalidConnection:
return nil, fmt.Errorf("invalid connection string: %s", rawAddress)
case tlsConnection:
canonicalAddress, err = extractHostFromRemoteURL(rawAddress) // Ensure the form: "host:port" ...
if err != nil {
return nil, err
}
certPool, err := systemCertPool()
if err != nil {
return nil, err
}
creds := credentials.NewClientTLSFromCert(certPool, "")
connOpts = append(connOpts, grpc.WithTransportCredentials(creds))
case tcpConnection:
canonicalAddress, err = extractHostFromRemoteURL(rawAddress) // Ensure the form: "host:port" ...
if err != nil {
return nil, err
}
connOpts = append(connOpts, grpc.WithInsecure())
case unixConnection:
canonicalAddress = rawAddress // This will be overriden by the custom dialer...
connOpts = append(
connOpts,
grpc.WithInsecure(),
grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
path, err := extractPathFromSocketURL(addr)
if err != nil {
return nil, err
}
return net.DialTimeout("unix", path, timeout)
}),
)
}
connOpts = append(connOpts,
grpc.WithDialer(func(a string, timeout time.Duration) (net.Conn, error) {
return net.DialTimeout(network, a, timeout)
}))
conn, err := grpc.Dial(addr, connOpts...)
conn, err := grpc.Dial(canonicalAddress, connOpts...)
if err != nil {
return nil, err
}
......@@ -34,28 +79,20 @@ func Dial(rawAddress string, connOpts []grpc.DialOption) (*grpc.ClientConn, erro
return conn, nil
}
func parseAddress(rawAddress string) (network, addr string, err error) {
// Parsing unix:// URL's with url.Parse does not give the result we want
// so we do it manually.
for _, prefix := range []string{"unix://", "unix:"} {
if strings.HasPrefix(rawAddress, prefix) {
return "unix", strings.TrimPrefix(rawAddress, prefix), nil
}
}
func getConnectionType(rawAddress string) connectionType {
u, err := url.Parse(rawAddress)
if err != nil {
return "", "", err
return invalidConnection
}
if u.Scheme != "tcp" {
return "", "", fmt.Errorf("unknown scheme: %q", rawAddress)
}
if u.Host == "" {
return "", "", fmt.Errorf("network tcp requires host: %q", rawAddress)
}
if u.Path != "" {
return "", "", fmt.Errorf("network tcp should have no path: %q", rawAddress)
switch u.Scheme {
case "tls":
return tlsConnection
case "unix":
return unixConnection
case "tcp":
return tcpConnection
default:
return invalidConnection
}
return "tcp", u.Host, nil
}
// +build !darwin
package client
import "crypto/x509"
// systemCertPool has an override on macOS.
func systemCertPool() (*x509.CertPool, error) { return x509.SystemCertPool() }
package client
import (
"crypto/x509"
"io/ioutil"
"os"
"path"
)
// systemCertPool circumvents the fact that Go on macOS does not support
// SSL_CERT_{DIR,FILE}.
func systemCertPool() (*x509.CertPool, error) {
var certPem []byte
if f := os.Getenv("SSL_CERT_FILE"); len(f) > 0 {
pem, err := ioutil.ReadFile(f)
if err != nil {
return nil, err
}
pem = append(pem, '\n')
certPem = append(certPem, pem...)
}
if d := os.Getenv("SSL_CERT_DIR"); len(d) > 0 {
entries, err := ioutil.ReadDir(d)
if err != nil {
return nil, err
}
for _, entry := range entries {
if entry.IsDir() {
continue
}
pem, err := ioutil.ReadFile(path.Join(d, entry.Name()))
if err != nil {
return nil, err
}
pem = append(pem, '\n')
certPem = append(certPem, pem...)
}
}
pool, err := x509.SystemCertPool()
if err != nil {
return nil, err
}
pool.AppendCertsFromPEM(certPem)
return pool, nil
}
package client
import (
"context"
"io"
"gitlab.com/gitlab-org/gitaly-proto/go/gitalypb"
"gitlab.com/gitlab-org/gitaly/streamio"
"golang.org/x/net/context"
"google.golang.org/grpc"
)
......
package client
import (
"context"
"io"
"gitlab.com/gitlab-org/gitaly-proto/go/gitalypb"
"gitlab.com/gitlab-org/gitaly/streamio"
"golang.org/x/net/context"
"google.golang.org/grpc"
)
......
package client
import (
"context"
"io"
"gitlab.com/gitlab-org/gitaly-proto/go/gitalypb"
"gitlab.com/gitlab-org/gitaly/streamio"
"golang.org/x/net/context"
"google.golang.org/grpc"
)
......
......@@ -405,12 +405,12 @@
"versionExact": "v0.120.0"
},
{
"checksumSHA1": "uvp/juXL0uZrtdQqcgXm4+COCO8=",
"checksumSHA1": "aL94A408/lEOF/XC4KDHvZnwnbk=",
"path": "gitlab.com/gitlab-org/gitaly/client",
"revision": "75643882c8c5e1abb46fe29b5c4ddac3cc0dffeb",
"revisionTime": "2018-10-04T18:41:07Z",
"version": "v0.124.0",
"versionExact": "v0.124.0"
"revision": "ef786ff72690836cc3ec1214b11bd10748271c5b",
"revisionTime": "2019-01-14T13:41:50Z",
"version": "v1.13.0",
"versionExact": "v1.13.0"
},
{
"checksumSHA1": "mifcYH0qXpoPkX5KzXoM3mterWQ=",
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment