Commit 1f6b19d8 authored by Nick Thomas's avatar Nick Thomas

Merge branch 'id-api-https' into 'master'

Support calling internal API using HTTPS

Closes #179

See merge request gitlab-org/gitlab-shell!297
parents 805bdd0d cffbe0eb
......@@ -6,7 +6,6 @@ import (
"os"
"path"
"path/filepath"
"strings"
yaml "gopkg.in/yaml.v2"
)
......@@ -26,6 +25,9 @@ type HttpSettingsConfig struct {
User string `yaml:"user"`
Password string `yaml:"password"`
ReadTimeoutSeconds uint64 `yaml:"read_timeout"`
CaFile string `yaml:"ca_file"`
CaPath string `yaml:"ca_path"`
SelfSignedCert bool `yaml:"self_signed_cert"`
}
type Config struct {
......@@ -59,10 +61,6 @@ func (c *Config) FeatureEnabled(featureName string) bool {
return false
}
if !strings.HasPrefix(c.GitlabUrl, "http+unix://") && !strings.HasPrefix(c.GitlabUrl, "http://") {
return false
}
for _, enabledFeature := range c.Migration.Features {
if enabledFeature == featureName {
return true
......
......@@ -94,6 +94,13 @@ func TestParseConfig(t *testing.T) {
secret: "default-secret-content",
httpSettings: HttpSettingsConfig{User: "user_basic_auth", Password: "password_basic_auth", ReadTimeoutSeconds: 500},
},
{
yaml: "http_settings:\n ca_file: /etc/ssl/cert.pem\n ca_path: /etc/pki/tls/certs\n self_signed_cert: true",
path: path.Join(testRoot, "gitlab-shell.log"),
format: "text",
secret: "default-secret-content",
httpSettings: HttpSettingsConfig{CaFile: "/etc/ssl/cert.pem", CaPath: "/etc/pki/tls/certs", SelfSignedCert: true},
},
}
for _, tc := range testCases {
......@@ -158,13 +165,13 @@ func TestFeatureEnabled(t *testing.T) {
expectEnabled: true,
},
{
desc: "When the protocol is not supported",
desc: "When the protocol is https and the feature enabled",
config: &Config{
GitlabUrl: "https://localhost:3000",
Migration: MigrationConfig{Enabled: true, Features: []string{"discover"}},
},
feature: "discover",
expectEnabled: false,
expectEnabled: true,
},
}
......
......@@ -2,16 +2,21 @@ package config
import (
"context"
"crypto/tls"
"crypto/x509"
"io/ioutil"
"net"
"net/http"
"path/filepath"
"strings"
"time"
)
const (
socketBaseUrl = "http://unix"
UnixSocketProtocol = "http+unix://"
HttpProtocol = "http://"
unixSocketProtocol = "http+unix://"
httpProtocol = "http://"
httpsProtocol = "https://"
defaultReadTimeoutSeconds = 300
)
......@@ -27,10 +32,12 @@ func (c *Config) GetHttpClient() *HttpClient {
var transport *http.Transport
var host string
if strings.HasPrefix(c.GitlabUrl, UnixSocketProtocol) {
if strings.HasPrefix(c.GitlabUrl, unixSocketProtocol) {
transport, host = c.buildSocketTransport()
} else if strings.HasPrefix(c.GitlabUrl, HttpProtocol) {
} else if strings.HasPrefix(c.GitlabUrl, httpProtocol) {
transport, host = c.buildHttpTransport()
} else if strings.HasPrefix(c.GitlabUrl, httpsProtocol) {
transport, host = c.buildHttpsTransport()
} else {
return nil
}
......@@ -48,7 +55,7 @@ func (c *Config) GetHttpClient() *HttpClient {
}
func (c *Config) buildSocketTransport() (*http.Transport, string) {
socketPath := strings.TrimPrefix(c.GitlabUrl, UnixSocketProtocol)
socketPath := strings.TrimPrefix(c.GitlabUrl, unixSocketProtocol)
transport := &http.Transport{
DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
dialer := net.Dialer{}
......@@ -59,6 +66,47 @@ func (c *Config) buildSocketTransport() (*http.Transport, string) {
return transport, socketBaseUrl
}
func (c *Config) buildHttpsTransport() (*http.Transport, string) {
certPool, err := x509.SystemCertPool()
if err != nil {
certPool = x509.NewCertPool()
}
caFile := c.HttpSettings.CaFile
if caFile != "" {
addCertToPool(certPool, caFile)
}
caPath := c.HttpSettings.CaPath
if caPath != "" {
fis, _ := ioutil.ReadDir(caPath)
for _, fi := range fis {
if fi.IsDir() {
continue
}
addCertToPool(certPool, filepath.Join(caPath, fi.Name()))
}
}
transport := &http.Transport{
TLSClientConfig: &tls.Config{
RootCAs: certPool,
InsecureSkipVerify: c.HttpSettings.SelfSignedCert,
},
}
return transport, c.GitlabUrl
}
func addCertToPool(certPool *x509.CertPool, fileName string) {
cert, err := ioutil.ReadFile(fileName)
if err == nil {
certPool.AppendCertsFromPEM(cert)
}
}
func (c *Config) buildHttpTransport() (*http.Transport, string) {
return &http.Transport{}, c.GitlabUrl
}
......
......@@ -6,15 +6,21 @@ import (
"fmt"
"io/ioutil"
"net/http"
"path"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitlab-shell/go/internal/config"
"gitlab.com/gitlab-org/gitlab-shell/go/internal/gitlabnet/testserver"
"gitlab.com/gitlab-org/gitlab-shell/go/internal/testhelper"
)
func TestClients(t *testing.T) {
testDirCleanup, err := testhelper.PrepareTestRootDir()
require.NoError(t, err)
defer testDirCleanup()
requests := []testserver.TestRequestHandler{
{
Path: "/api/v4/internal/hello",
......@@ -64,19 +70,26 @@ func TestClients(t *testing.T) {
testCases := []struct {
desc string
secret string
config *config.Config
server func([]testserver.TestRequestHandler) (func(), string, error)
}{
{
desc: "Socket client",
secret: "sssh, it's a secret",
config: &config.Config{},
server: testserver.StartSocketHttpServer,
},
{
desc: "Http client",
secret: "sssh, it's a secret",
config: &config.Config{},
server: testserver.StartHttpServer,
},
{
desc: "Https client",
config: &config.Config{
HttpSettings: config.HttpSettingsConfig{CaFile: path.Join(testhelper.TestRoot, "certs/valid/server.crt")},
},
server: testserver.StartHttpsServer,
},
}
for _, tc := range testCases {
......@@ -85,7 +98,10 @@ func TestClients(t *testing.T) {
defer cleanup()
require.NoError(t, err)
client, err := GetClient(&config.Config{GitlabUrl: url, Secret: tc.secret})
tc.config.GitlabUrl = url
tc.config.Secret = "sssh, it's a secret"
client, err := GetClient(tc.config)
require.NoError(t, err)
testBrokenRequest(t, client)
......
package gitlabnet
import (
"fmt"
"io/ioutil"
"net/http"
"path"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitlab-shell/go/internal/config"
"gitlab.com/gitlab-org/gitlab-shell/go/internal/gitlabnet/testserver"
"gitlab.com/gitlab-org/gitlab-shell/go/internal/testhelper"
)
func TestSuccessfulRequests(t *testing.T) {
testCases := []struct {
desc string
config *config.Config
}{
{
desc: "Valid CaFile",
config: &config.Config{
HttpSettings: config.HttpSettingsConfig{CaFile: path.Join(testhelper.TestRoot, "certs/valid/server.crt")},
},
},
{
desc: "Valid CaPath",
config: &config.Config{
HttpSettings: config.HttpSettingsConfig{CaPath: path.Join(testhelper.TestRoot, "certs/valid")},
},
},
{
desc: "Self signed cert option enabled",
config: &config.Config{
HttpSettings: config.HttpSettingsConfig{SelfSignedCert: true},
},
},
{
desc: "Invalid cert with self signed cert option enabled",
config: &config.Config{
HttpSettings: config.HttpSettingsConfig{SelfSignedCert: true, CaFile: path.Join(testhelper.TestRoot, "certs/valid/server.crt")},
},
},
}
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
client, cleanup := setupWithRequests(t, tc.config)
defer cleanup()
response, err := client.Get("/hello")
require.NoError(t, err)
require.NotNil(t, response)
defer response.Body.Close()
responseBody, err := ioutil.ReadAll(response.Body)
assert.NoError(t, err)
assert.Equal(t, string(responseBody), "Hello")
})
}
}
func TestFailedRequests(t *testing.T) {
testCases := []struct {
desc string
config *config.Config
}{
{
desc: "Invalid CaFile",
config: &config.Config{
HttpSettings: config.HttpSettingsConfig{CaFile: path.Join(testhelper.TestRoot, "certs/invalid/server.crt")},
},
},
{
desc: "Invalid CaPath",
config: &config.Config{
HttpSettings: config.HttpSettingsConfig{CaPath: path.Join(testhelper.TestRoot, "certs/invalid")},
},
},
{
desc: "Empty config",
config: &config.Config{},
},
}
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
client, cleanup := setupWithRequests(t, tc.config)
defer cleanup()
_, err := client.Get("/hello")
require.Error(t, err)
assert.Equal(t, err.Error(), "Internal API unreachable")
})
}
}
func setupWithRequests(t *testing.T, config *config.Config) (*GitlabClient, func()) {
testDirCleanup, err := testhelper.PrepareTestRootDir()
require.NoError(t, err)
defer testDirCleanup()
requests := []testserver.TestRequestHandler{
{
Path: "/api/v4/internal/hello",
Handler: func(w http.ResponseWriter, r *http.Request) {
require.Equal(t, http.MethodGet, r.Method)
fmt.Fprint(w, "Hello")
},
},
}
cleanup, url, err := testserver.StartHttpsServer(requests)
require.NoError(t, err)
config.GitlabUrl = url
client, err := GetClient(config)
require.NoError(t, err)
return client, cleanup
}
package testserver
import (
"crypto/tls"
"io/ioutil"
"log"
"net"
......@@ -9,6 +10,8 @@ import (
"os"
"path"
"path/filepath"
"gitlab.com/gitlab-org/gitlab-shell/go/internal/testhelper"
)
var (
......@@ -50,6 +53,23 @@ func StartHttpServer(handlers []TestRequestHandler) (func(), string, error) {
return server.Close, server.URL, nil
}
func StartHttpsServer(handlers []TestRequestHandler) (func(), string, error) {
crt := path.Join(testhelper.TestRoot, "certs/valid/server.crt")
key := path.Join(testhelper.TestRoot, "certs/valid/server.key")
server := httptest.NewUnstartedServer(buildHandler(handlers))
cer, err := tls.LoadX509KeyPair(crt, key)
if err != nil {
return nil, "", err
}
server.TLS = &tls.Config{Certificates: []tls.Certificate{cer}}
server.StartTLS()
return server.Close, server.URL, nil
}
func cleanupSocket() {
os.RemoveAll(tempDir)
}
......
-----BEGIN CERTIFICATE-----
MinvalidcertAOvHjs6cs1R9MAoGCCqGSM49BAMCMBQxEjAQBgNVBAMMCWxvY2Fs
ainvalidcertOTA0MjQxNjM4NTBaFw0yOTA0MjExNjM4NTBaMBQxEjAQBgNVBAMM
CinvalidcertdDB2MBAGByqGSM49AgEGBSuBBAAiA2IABJ5m7oW9OuL7aTAC04sL
3invalidcertdB2L0GsVCImav4PEpx6UAjkoiNGW9j0zPdNgxTYDjiCaGmr1aY2X
kinvalidcert7MNq7H8v7Ce/vrKkcDMOX8Gd/ddT3dEVqzAKBggqhkjOPQQDAgNp
AinvalidcertswcyjiB+A+ZjMSfaOsA2hAP0I3fkTcry386DePViMfnaIjm7rcuu
Jinvalidcert5V5CHypOxio1tOtGjaDkSH2FCdoatMyIe02+F6TIo44i4J/zjN52
Jinvalidcert
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIIBsDCCAVqgAwIBAgIJALlYxbvxYURTMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV
BAMMCWxvY2FsaG9zdDAeFw0xOTA1MDEwNzM2MDNaFw0yOTA0MjgwNzM2MDNaMBQx
EjAQBgNVBAMMCWxvY2FsaG9zdDBcMA0GCSqGSIb3DQEBAQUAA0sAMEgCQQDlgT4b
0ufjR6mIaTuP4lXXtY74YlHDlh4m/qqD4aTLPU/x6fMvvqsqIqKX9emAz/U5rzXv
GKwpN2DlBKEADzWzAgMBAAGjgY4wgYswHQYDVR0OBBYEFM13Hmhp8oMJfntPBpO6
fDJ6e1W0MB8GA1UdIwQYMBaAFM13Hmhp8oMJfntPBpO6fDJ6e1W0MA4GA1UdDwEB
/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwGgYDVR0RBBMw
EYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEBCwUAA0EAohV1Gh0A6Q5SwY3I
I5Dfu3INMU/9Id+EoHorJwkN6oUuRMjwdwjzZaa1WLWlkw7EHYwk1YC3qltKnDp/
PuFxwQ==
-----END CERTIFICATE-----
-----BEGIN PRIVATE KEY-----
MIIBVwIBADANBgkqhkiG9w0BAQEFAASCAUEwggE9AgEAAkEA5YE+G9Ln40epiGk7
j+JV17WO+GJRw5YeJv6qg+Gkyz1P8enzL76rKiKil/XpgM/1Oa817xisKTdg5QSh
AA81swIDAQABAkEAofMPhsbPB1y8TxwjAadvd+YQW0nV9LRr1oyCesmxZhAp0KQq
3vLs24nQAYH7mAtfcpv0l1t8NHl6JajZd3sLIQIhAPLQ4HuZSGPTncJFZv4/tqwf
uXmouo9Fo7+eAXXx1H5LAiEA8fdXw/vuUzb+3I/hwkS5i1zgsYdMye63UffH+aSS
5TkCIQC1ZskebaCAO7szROgx7+WH59eIBT8DBFLWN7P9qmJGywIhALPDL7gRxgia
thPE7VN33WUVNnWN8FWhfR5veGkWhG+5AiEAnxDUMc2ZewuNkPZHZAIobRAily+q
Q5REf1FU92ptvbg=
-----END PRIVATE KEY-----
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment