Commit 88980664 authored by Nimi Wariboko Jr's avatar Nimi Wariboko Jr Committed by GitHub

Merge branch 'master' into proxy/single-webconn

parents 6e9439d2 fffc1bed
language: go
go:
- 1.6.2
- 1.6.3
- tip
env:
- CGO_ENABLED=0
before_install:
# Decrypts a script that installs an authenticated cookie
# for git to use when cloning from googlesource.com.
......@@ -24,7 +21,7 @@ script:
- diff <(echo -n) <(gofmt -s -d .)
- ineffassign .
- go vet ./...
- go test ./...
- go test -race ./...
after_script:
- golint ./...
......@@ -6,12 +6,11 @@ clone_folder: c:\gopath\src\github.com\mholt\caddy
environment:
GOPATH: c:\gopath
CGO_ENABLED: 0
install:
- rmdir c:\go /s /q
- appveyor DownloadFile https://storage.googleapis.com/golang/go1.6.2.windows-amd64.zip
- 7z x go1.6.2.windows-amd64.zip -y -oC:\ > NUL
- appveyor DownloadFile https://storage.googleapis.com/golang/go1.6.3.windows-amd64.zip
- 7z x go1.6.3.windows-amd64.zip -y -oC:\ > NUL
- go version
- go env
- go get -t ./...
......@@ -23,7 +22,7 @@ build: off
test_script:
- go vet ./...
- go test ./...
- go test -race ./...
- ineffassign .
after_test:
......
......@@ -21,8 +21,6 @@ import (
"log"
"net"
"os"
"os/exec"
"runtime"
"strconv"
"strings"
"sync"
......@@ -725,24 +723,6 @@ func IsLoopback(addr string) bool {
strings.HasPrefix(host, "127.")
}
// checkFdlimit issues a warning if the OS limit for
// max file descriptors is below a recommended minimum.
func checkFdlimit() {
const min = 8192
// Warn if ulimit is too low for production sites
if runtime.GOOS == "linux" || runtime.GOOS == "darwin" {
out, err := exec.Command("sh", "-c", "ulimit -n").Output() // use sh because ulimit isn't in Linux $PATH
if err == nil {
lim, err := strconv.Atoi(string(bytes.TrimSpace(out)))
if err == nil && lim < min {
fmt.Printf("WARNING: File descriptor limit %d is too low for production servers. "+
"At least %d is recommended. Fix with \"ulimit -n %d\".\n", lim, min, min)
}
}
}
}
// Upgrade re-launches the process, preserving the listeners
// for a graceful restart. It does NOT load new configuration;
// it only starts the process anew with a fresh binary.
......
package proxy
import (
"hash/fnv"
"math"
"math/rand"
"net"
"net/http"
"sync"
)
......@@ -11,20 +14,21 @@ type HostPool []*UpstreamHost
// Policy decides how a host will be selected from a pool.
type Policy interface {
Select(pool HostPool) *UpstreamHost
Select(pool HostPool, r *http.Request) *UpstreamHost
}
func init() {
RegisterPolicy("random", func() Policy { return &Random{} })
RegisterPolicy("least_conn", func() Policy { return &LeastConn{} })
RegisterPolicy("round_robin", func() Policy { return &RoundRobin{} })
RegisterPolicy("ip_hash", func() Policy { return &IPHash{} })
}
// Random is a policy that selects up hosts from a pool at random.
type Random struct{}
// Select selects an up host at random from the specified pool.
func (r *Random) Select(pool HostPool) *UpstreamHost {
func (r *Random) Select(pool HostPool, request *http.Request) *UpstreamHost {
// Because the number of available hosts isn't known
// up front, the host is selected via reservoir sampling
......@@ -53,7 +57,7 @@ type LeastConn struct{}
// Select selects the up host with the least number of connections in the
// pool. If more than one host has the same least number of connections,
// one of the hosts is chosen at random.
func (r *LeastConn) Select(pool HostPool) *UpstreamHost {
func (r *LeastConn) Select(pool HostPool, request *http.Request) *UpstreamHost {
var bestHost *UpstreamHost
count := 0
leastConn := int64(math.MaxInt64)
......@@ -86,7 +90,7 @@ type RoundRobin struct {
}
// Select selects an up host from the pool using a round robin ordering scheme.
func (r *RoundRobin) Select(pool HostPool) *UpstreamHost {
func (r *RoundRobin) Select(pool HostPool, request *http.Request) *UpstreamHost {
poolLen := uint32(len(pool))
r.mutex.Lock()
defer r.mutex.Unlock()
......@@ -100,3 +104,35 @@ func (r *RoundRobin) Select(pool HostPool) *UpstreamHost {
}
return nil
}
// IPHash is a policy that selects hosts based on hashing the request ip
type IPHash struct{}
func hash(s string) uint32 {
h := fnv.New32a()
h.Write([]byte(s))
return h.Sum32()
}
// Select selects an up host from the pool using a round robin ordering scheme.
func (r *IPHash) Select(pool HostPool, request *http.Request) *UpstreamHost {
poolLen := uint32(len(pool))
clientIP, _, err := net.SplitHostPort(request.RemoteAddr)
if err != nil {
clientIP = request.RemoteAddr
}
hash := hash(clientIP)
for {
if poolLen == 0 {
break
}
index := hash % poolLen
host := pool[index]
if host.Available() {
return host
}
pool = append(pool[:index], pool[index+1:]...)
poolLen--
}
return nil
}
......@@ -21,7 +21,7 @@ func TestMain(m *testing.M) {
type customPolicy struct{}
func (r *customPolicy) Select(pool HostPool) *UpstreamHost {
func (r *customPolicy) Select(pool HostPool, request *http.Request) *UpstreamHost {
return pool[0]
}
......@@ -43,37 +43,39 @@ func testPool() HostPool {
func TestRoundRobinPolicy(t *testing.T) {
pool := testPool()
rrPolicy := &RoundRobin{}
h := rrPolicy.Select(pool)
request, _ := http.NewRequest("GET", "/", nil)
h := rrPolicy.Select(pool, request)
// First selected host is 1, because counter starts at 0
// and increments before host is selected
if h != pool[1] {
t.Error("Expected first round robin host to be second host in the pool.")
}
h = rrPolicy.Select(pool)
h = rrPolicy.Select(pool, request)
if h != pool[2] {
t.Error("Expected second round robin host to be third host in the pool.")
}
h = rrPolicy.Select(pool)
h = rrPolicy.Select(pool, request)
if h != pool[0] {
t.Error("Expected third round robin host to be first host in the pool.")
}
// mark host as down
pool[1].Unhealthy = true
h = rrPolicy.Select(pool)
h = rrPolicy.Select(pool, request)
if h != pool[2] {
t.Error("Expected to skip down host.")
}
// mark host as up
pool[1].Unhealthy = false
h = rrPolicy.Select(pool)
h = rrPolicy.Select(pool, request)
if h == pool[2] {
t.Error("Expected to balance evenly among healthy hosts")
}
// mark host as full
pool[1].Conns = 1
pool[1].MaxConns = 1
h = rrPolicy.Select(pool)
h = rrPolicy.Select(pool, request)
if h != pool[2] {
t.Error("Expected to skip full host.")
}
......@@ -82,14 +84,16 @@ func TestRoundRobinPolicy(t *testing.T) {
func TestLeastConnPolicy(t *testing.T) {
pool := testPool()
lcPolicy := &LeastConn{}
request, _ := http.NewRequest("GET", "/", nil)
pool[0].Conns = 10
pool[1].Conns = 10
h := lcPolicy.Select(pool)
h := lcPolicy.Select(pool, request)
if h != pool[2] {
t.Error("Expected least connection host to be third host.")
}
pool[2].Conns = 100
h = lcPolicy.Select(pool)
h = lcPolicy.Select(pool, request)
if h != pool[0] && h != pool[1] {
t.Error("Expected least connection host to be first or second host.")
}
......@@ -98,8 +102,127 @@ func TestLeastConnPolicy(t *testing.T) {
func TestCustomPolicy(t *testing.T) {
pool := testPool()
customPolicy := &customPolicy{}
h := customPolicy.Select(pool)
request, _ := http.NewRequest("GET", "/", nil)
h := customPolicy.Select(pool, request)
if h != pool[0] {
t.Error("Expected custom policy host to be the first host.")
}
}
func TestIPHashPolicy(t *testing.T) {
pool := testPool()
ipHash := &IPHash{}
request, _ := http.NewRequest("GET", "/", nil)
// We should be able to predict where every request is routed.
request.RemoteAddr = "172.0.0.1:80"
h := ipHash.Select(pool, request)
if h != pool[1] {
t.Error("Expected ip hash policy host to be the second host.")
}
request.RemoteAddr = "172.0.0.2:80"
h = ipHash.Select(pool, request)
if h != pool[1] {
t.Error("Expected ip hash policy host to be the second host.")
}
request.RemoteAddr = "172.0.0.3:80"
h = ipHash.Select(pool, request)
if h != pool[2] {
t.Error("Expected ip hash policy host to be the third host.")
}
request.RemoteAddr = "172.0.0.4:80"
h = ipHash.Select(pool, request)
if h != pool[1] {
t.Error("Expected ip hash policy host to be the second host.")
}
// we should get the same results without a port
request.RemoteAddr = "172.0.0.1"
h = ipHash.Select(pool, request)
if h != pool[1] {
t.Error("Expected ip hash policy host to be the second host.")
}
request.RemoteAddr = "172.0.0.2"
h = ipHash.Select(pool, request)
if h != pool[1] {
t.Error("Expected ip hash policy host to be the second host.")
}
request.RemoteAddr = "172.0.0.3"
h = ipHash.Select(pool, request)
if h != pool[2] {
t.Error("Expected ip hash policy host to be the third host.")
}
request.RemoteAddr = "172.0.0.4"
h = ipHash.Select(pool, request)
if h != pool[1] {
t.Error("Expected ip hash policy host to be the second host.")
}
// we should get a healthy host if the original host is unhealthy and a
// healthy host is available
request.RemoteAddr = "172.0.0.1"
pool[1].Unhealthy = true
h = ipHash.Select(pool, request)
if h != pool[0] {
t.Error("Expected ip hash policy host to be the first host.")
}
request.RemoteAddr = "172.0.0.2"
h = ipHash.Select(pool, request)
if h != pool[1] {
t.Error("Expected ip hash policy host to be the second host.")
}
pool[1].Unhealthy = false
request.RemoteAddr = "172.0.0.3"
pool[2].Unhealthy = true
h = ipHash.Select(pool, request)
if h != pool[0] {
t.Error("Expected ip hash policy host to be the first host.")
}
request.RemoteAddr = "172.0.0.4"
h = ipHash.Select(pool, request)
if h != pool[0] {
t.Error("Expected ip hash policy host to be the first host.")
}
// We should be able to resize the host pool and still be able to predict
// where a request will be routed with the same IP's used above
pool = []*UpstreamHost{
{
Name: workableServer.URL, // this should resolve (healthcheck test)
},
{
Name: "http://localhost:99998", // this shouldn't
},
}
pool = HostPool(pool)
request.RemoteAddr = "172.0.0.1:80"
h = ipHash.Select(pool, request)
if h != pool[0] {
t.Error("Expected ip hash policy host to be the first host.")
}
request.RemoteAddr = "172.0.0.2:80"
h = ipHash.Select(pool, request)
if h != pool[1] {
t.Error("Expected ip hash policy host to be the second host.")
}
request.RemoteAddr = "172.0.0.3:80"
h = ipHash.Select(pool, request)
if h != pool[0] {
t.Error("Expected ip hash policy host to be the first host.")
}
request.RemoteAddr = "172.0.0.4:80"
h = ipHash.Select(pool, request)
if h != pool[1] {
t.Error("Expected ip hash policy host to be the second host.")
}
// We should get nil when there are no healthy hosts
pool[0].Unhealthy = true
pool[1].Unhealthy = true
h = ipHash.Select(pool, request)
if h != nil {
t.Error("Expected ip hash policy host to be nil.")
}
}
......@@ -27,7 +27,7 @@ type Upstream interface {
// The path this upstream host should be routed on
From() string
// Selects an upstream host to be routed to.
Select() *UpstreamHost
Select(*http.Request) *UpstreamHost
// Checks if subpath is not an ignored path
AllowedPath(string) bool
}
......@@ -93,7 +93,7 @@ func (p Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) {
// hosts until timeout (or until we get a nil host).
start := time.Now()
for time.Now().Sub(start) < tryDuration {
host := upstream.Select()
host := upstream.Select(r)
if host == nil {
return http.StatusBadGateway, errUnreachable
}
......@@ -108,7 +108,7 @@ func (p Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) {
if nameURL, err := url.Parse(host.Name); err == nil {
outreq.Host = nameURL.Host
if proxy == nil {
proxy = NewSingleHostReverseProxy(nameURL, host.WithoutPathPrefix, 0)
proxy = NewSingleHostReverseProxy(nameURL, host.WithoutPathPrefix, http.DefaultMaxIdleConnsPerHost)
}
// use upstream credentials by default
......
......@@ -362,9 +362,11 @@ func TestUpstreamHeadersUpdate(t *testing.T) {
defer log.SetOutput(os.Stderr)
var actualHeaders http.Header
var actualHost string
backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("Hello, client"))
actualHeaders = r.Header
actualHost = r.Host
}))
defer backend.Close()
......@@ -376,6 +378,7 @@ func TestUpstreamHeadersUpdate(t *testing.T) {
"+Add-Me": {"Add-Value"},
"-Remove-Me": {""},
"Replace-Me": {"{hostname}"},
"Host": {"{>Host}"},
}
// set up proxy
p := &Proxy{
......@@ -390,10 +393,12 @@ func TestUpstreamHeadersUpdate(t *testing.T) {
}
w := httptest.NewRecorder()
const expectHost = "example.com"
//add initial headers
r.Header.Add("Merge-Me", "Initial")
r.Header.Add("Remove-Me", "Remove-Value")
r.Header.Add("Replace-Me", "Replace-Value")
r.Header.Add("Host", expectHost)
p.ServeHTTP(w, r)
......@@ -426,6 +431,10 @@ func TestUpstreamHeadersUpdate(t *testing.T) {
t.Errorf("Request sent to upstream backend should replace value of %v header with %v. Instead value was %v", headerKey, headerValue, value)
}
if actualHost != expectHost {
t.Errorf("Request sent to upstream backend should have value of Host with %s, but got %s", expectHost, actualHost)
}
}
func TestDownstreamHeadersUpdate(t *testing.T) {
......@@ -721,7 +730,7 @@ func newFakeUpstream(name string, insecure bool) *fakeUpstream {
from: "/",
host: &UpstreamHost{
Name: name,
ReverseProxy: NewSingleHostReverseProxy(uri, "", 0),
ReverseProxy: NewSingleHostReverseProxy(uri, "", http.DefaultMaxIdleConnsPerHost),
},
}
if insecure {
......@@ -741,7 +750,7 @@ func (u *fakeUpstream) From() string {
return u.from
}
func (u *fakeUpstream) Select() *UpstreamHost {
func (u *fakeUpstream) Select(r *http.Request) *UpstreamHost {
if u.host == nil {
uri, err := url.Parse(u.name)
if err != nil {
......@@ -749,7 +758,7 @@ func (u *fakeUpstream) Select() *UpstreamHost {
}
u.host = &UpstreamHost{
Name: u.name,
ReverseProxy: NewSingleHostReverseProxy(uri, u.without, 0),
ReverseProxy: NewSingleHostReverseProxy(uri, u.without, http.DefaultMaxIdleConnsPerHost),
}
}
return u.host
......@@ -786,11 +795,11 @@ func (u *fakeWsUpstream) From() string {
return "/"
}
func (u *fakeWsUpstream) Select() *UpstreamHost {
func (u *fakeWsUpstream) Select(r *http.Request) *UpstreamHost {
uri, _ := url.Parse(u.name)
return &UpstreamHost{
Name: u.name,
ReverseProxy: NewSingleHostReverseProxy(uri, u.without, 0),
ReverseProxy: NewSingleHostReverseProxy(uri, u.without, http.DefaultMaxIdleConnsPerHost),
UpstreamHeaders: http.Header{
"Connection": {"{>Connection}"},
"Upgrade": {"{>Upgrade}"}},
......
......@@ -122,7 +122,10 @@ func NewSingleHostReverseProxy(target *url.URL, without string, keepalive int) *
rp.Transport = &http.Transport{
Dial: socketDial(target.String()),
}
} else if keepalive != 0 {
} else if keepalive != http.DefaultMaxIdleConnsPerHost {
// if keepalive is equal to the default,
// just use default transport, to avoid creating
// a brand new transport
rp.Transport = &http.Transport{
Proxy: http.ProxyFromEnvironment,
Dial: (&net.Dialer{
......@@ -132,7 +135,7 @@ func NewSingleHostReverseProxy(target *url.URL, without string, keepalive int) *
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
}
if keepalive < 0 {
if keepalive == 0 {
rp.Transport.(*http.Transport).DisableKeepAlives = true
} else {
rp.Transport.(*http.Transport).MaxIdleConnsPerHost = keepalive
......
......@@ -55,6 +55,7 @@ func NewStaticUpstreams(c caddyfile.Dispenser) ([]Upstream, error) {
FailTimeout: 10 * time.Second,
MaxFails: 1,
MaxConns: 0,
KeepAlive: http.DefaultMaxIdleConnsPerHost,
}
if !c.Args(&upstream.from) {
......@@ -321,6 +322,9 @@ func parseBlock(c *caddyfile.Dispenser, u *staticUpstream) error {
if err != nil {
return err
}
if n < 0 {
return c.ArgErr()
}
u.KeepAlive = n
default:
return c.Errf("unknown property '%s'", c.Val())
......@@ -356,7 +360,7 @@ func (u *staticUpstream) HealthCheckWorker(stop chan struct{}) {
}
}
func (u *staticUpstream) Select() *UpstreamHost {
func (u *staticUpstream) Select(r *http.Request) *UpstreamHost {
pool := u.Hosts
if len(pool) == 1 {
if !pool[0].Available() {
......@@ -374,11 +378,10 @@ func (u *staticUpstream) Select() *UpstreamHost {
if allUnavailable {
return nil
}
if u.Policy == nil {
return (&Random{}).Select(pool)
return (&Random{}).Select(pool, r)
}
return u.Policy.Select(pool)
return u.Policy.Select(pool, r)
}
func (u *staticUpstream) AllowedPath(requestPath string) bool {
......
package proxy
import (
"github.com/mholt/caddy/caddyfile"
"net/http"
"strings"
"testing"
"time"
"github.com/mholt/caddy/caddyfile"
)
func TestNewHost(t *testing.T) {
......@@ -72,14 +72,15 @@ func TestSelect(t *testing.T) {
FailTimeout: 10 * time.Second,
MaxFails: 1,
}
r, _ := http.NewRequest("GET", "/", nil)
upstream.Hosts[0].Unhealthy = true
upstream.Hosts[1].Unhealthy = true
upstream.Hosts[2].Unhealthy = true
if h := upstream.Select(); h != nil {
if h := upstream.Select(r); h != nil {
t.Error("Expected select to return nil as all host are down")
}
upstream.Hosts[2].Unhealthy = false
if h := upstream.Select(); h == nil {
if h := upstream.Select(r); h == nil {
t.Error("Expected select to not return nil")
}
upstream.Hosts[0].Conns = 1
......@@ -88,11 +89,11 @@ func TestSelect(t *testing.T) {
upstream.Hosts[1].MaxConns = 1
upstream.Hosts[2].Conns = 1
upstream.Hosts[2].MaxConns = 1
if h := upstream.Select(); h != nil {
if h := upstream.Select(r); h != nil {
t.Error("Expected select to return nil as all hosts are full")
}
upstream.Hosts[2].Conns = 0
if h := upstream.Select(); h == nil {
if h := upstream.Select(r); h == nil {
t.Error("Expected select to not return nil")
}
}
......@@ -188,6 +189,7 @@ func TestParseBlockHealthCheck(t *testing.T) {
}
func TestParseBlock(t *testing.T) {
r, _ := http.NewRequest("GET", "/", nil)
tests := []struct {
config string
}{
......@@ -207,7 +209,7 @@ func TestParseBlock(t *testing.T) {
t.Error("Expected no error. Got:", err.Error())
}
for _, upstream := range upstreams {
headers := upstream.Select().UpstreamHeaders
headers := upstream.Select(r).UpstreamHeaders
if _, ok := headers["Host"]; !ok {
t.Errorf("Test %d: Could not find the Host header", i+1)
......
......@@ -79,19 +79,22 @@ func PrivateKeyBytes(key crypto.PrivateKey) []byte {
}
func TestStandaloneTLSTicketKeyRotation(t *testing.T) {
type syncPkt struct {
ticketKey [32]byte
keysInUse int
}
tlsGovChan := make(chan struct{})
defer close(tlsGovChan)
callSync := make(chan bool, 1)
callSync := make(chan *syncPkt, 1)
defer close(callSync)
oldHook := setSessionTicketKeysTestHook
defer func() {
setSessionTicketKeysTestHook = oldHook
}()
var keysInUse [][32]byte
setSessionTicketKeysTestHook = func(keys [][32]byte) [][32]byte {
keysInUse = keys
callSync <- true
callSync <- &syncPkt{keys[0], len(keys)}
return keys
}
......@@ -104,17 +107,17 @@ func TestStandaloneTLSTicketKeyRotation(t *testing.T) {
var lastTicketKey [32]byte
for {
select {
case <-callSync:
if lastTicketKey == keysInUse[0] {
case pkt := <-callSync:
if lastTicketKey == pkt.ticketKey {
close(tlsGovChan)
t.Errorf("The same TLS ticket key has been used again (not rotated): %x.", lastTicketKey)
return
}
lastTicketKey = keysInUse[0]
lastTicketKey = pkt.ticketKey
rounds++
if rounds <= NumTickets && len(keysInUse) != rounds {
if rounds <= NumTickets && pkt.keysInUse != rounds {
close(tlsGovChan)
t.Errorf("Expected TLS ticket keys in use: %d; Got instead: %d.", rounds, len(keysInUse))
t.Errorf("Expected TLS ticket keys in use: %d; Got instead: %d.", rounds, pkt.keysInUse)
return
}
if c.SessionTicketsDisabled == true {
......
......@@ -20,9 +20,9 @@ DAEMONUSER=www-data
PIDFILE=/var/run/$NAME.pid
LOGFILE=/var/log/$NAME.log
CONFIGFILE=/etc/caddy/Caddyfile
DAEMONOPTS="-agree=true --pidfile=$PIDFILE log=$LOGFILE -conf=$CONFIGFILE"
DAEMONOPTS="-agree=true -pidfile=$PIDFILE -log=$LOGFILE -conf=$CONFIGFILE"
USERBIND="$(which setcap) cap_net_bind_service=+ep"
USERBIND="setcap cap_net_bind_service=+ep"
STOP_SCHEDULE="${STOP_SCHEDULE:-QUIT/5/TERM/5/KILL/5}"
test -x $DAEMON || exit 0
......@@ -37,12 +37,13 @@ ulimit -n 8192
start() {
$USERBIND $DAEMON
start-stop-daemon --start --quiet --make-pidfile --pidfile $PIDFILE \
--background --chuid $DAEMONUSER --exec $DAEMON -- $DAEMONOPTS
--background --chuid $DAEMONUSER --oknodo --exec $DAEMON -- $DAEMONOPTS
}
stop() {
start-stop-daemon --stop --quiet --remove-pidfile --pidfile $PIDFILE \
--retry=$STOP_SCHEDULE --name $NAME --oknodo
start-stop-daemon --stop --quiet --pidfile $PIDFILE --retry=$STOP_SCHEDULE \
--name $NAME --oknodo
rm -f $PIDFILE
}
reload() {
......
// +build !windows
package caddy
import (
"fmt"
"syscall"
)
// checkFdlimit issues a warning if the OS limit for
// max file descriptors is below a recommended minimum.
func checkFdlimit() {
const min = 8192
// Warn if ulimit is too low for production sites
rlimit := &syscall.Rlimit{}
err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, rlimit)
if err == nil && rlimit.Cur < min {
fmt.Printf("WARNING: File descriptor limit %d is too low for production servers. "+
"At least %d is recommended. Fix with \"ulimit -n %d\".\n", rlimit.Cur, min, min)
}
}
package caddy
// checkFdlimit issues a warning if the OS limit for
// max file descriptors is below a recommended minimum.
func checkFdlimit() {
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment