Skip to content

N
neo
Commit af03f483 authored by Levin Zimmermann's avatar Levin Zimmermann
1 Browse files
Options

go/client/zurl: Sync format to py upstream 

NEO/go and NEO/py zurl format diverged over time:

- kirr/neo@8c974485

However with nexedi/neoppod!21 a
common solution was found. From there, this patch aims to adjust NEO/go zurl
format to be in sync with NEO/py zurl format again.
parent 1ad088c8
Show whitespace changes
Inline Side-by-side
Showing with 66 additions and 68 deletions
go/neo/client.go
View file @ af03f483
// Copyright (C) 2017-2023 Nexedi SA and Contributors.
// Copyright (C) 2017-2024 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -484,7 +484,7 @@ func openClientByURL(ctx context.Context, u *url.URL, opt *zodb.DriverOptions) (
// parseURL extracts information from a NEO URI and puts this information into
// a urlInfo.
func parseURL(ctx context.Context, u *url.URL) (urlinfo *urlInfo, err error) {
// neo(s)://[credentials@]master1,master2,...,masterN/name?options
// neo(s)://name@master1,master2,...,masterN?options
var ssl bool
switch u.Scheme {
......@@ -493,50 +493,35 @@ func parseURL(ctx context.Context, u *url.URL) (urlinfo *urlInfo, err error) {
default: return nil, fmt.Errorf("invalid scheme")
}
cred := u.User.String()
// ca=ca.crt;cert=my.crt;key=my.key
cred = strings.ReplaceAll(cred, ";", "&") // ; is no longer in default separators set https://github.com/golang/go/issues/25192
x, err := xurl.ParseQuery(cred)
name := u.User.String()
if name == "" {
return nil, fmt.Errorf("cluster name not specified")
}
q, err := xurl.ParseQuery(u.RawQuery)
if err != nil {
return nil, fmt.Errorf("credentials: %s", err)
return nil, err
}
// xpop pops k from credentials, defaulting to $NEO_<K> if envok.
xpop := func(k string, envok bool) string {
v, ok := x[k]
// qpop pops k from query, defaulting to $NEO_<K> if envok.
qpop := func(k string, envok bool) string {
v, ok := q[k]
if !ok && envok {
v = os.Getenv("NEO_"+strings.ToUpper(k))
}
delete(x, k)
delete(q, k)
return v
}
netcfg := neonet.Config{}
netcfg.LoNode = xpop("lonode", false)
if !ssl {
if len(x) != 0 {
return nil, fmt.Errorf("credentials can be specified only with neos:// scheme")
}
} else {
netcfg.CA = xpop("ca", true)
netcfg.Cert = xpop("cert", true)
netcfg.Key = xpop("key", true)
if len(x) != 0 {
return nil, fmt.Errorf("invalid credentials: %v", x)
}
}
name := u.Path
name = strings.TrimPrefix(name, "/")
if name == "" {
return nil, fmt.Errorf("cluster name not specified")
}
q, err := xurl.ParseQuery(u.RawQuery)
if err != nil {
return nil, err
}
netcfg.LoNode = qpop("lonode", false)
// neos:// force TLS to be used and take ca/cert/key from environment if
// TLS credentials are not explicitly specified in uri
// neo:// use TLS only if ca/cert/key are explicitly specified in uri
netcfg.CA = qpop("ca", ssl)
netcfg.Cert = qpop("cert", ssl)
netcfg.Key = qpop("key", ssl)
// pop not yet used client options
// (our neo client doesn't apply their effect yet)
......
go/neo/client_test.go
View file @ af03f483
// Copyright (C) 2020-2023 Nexedi SA and Contributors.
// Copyright (C) 2020-2024 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -175,7 +175,7 @@ func (n *NEOPySrv) clusterName() string {
}
func (n *NEOPySrv) URL() string {
return fmt.Sprintf("%s%s/%s", n.opt.URLPrefix(), strings.Join(n.masterAddrSlice, ","), n.clusterName())
return fmt.Sprintf("%s%s%s", n.opt.URLPrefix(), strings.Join(n.masterAddrSlice, ","), n.opt.URLQuery())
}
func (n *NEOPySrv) LogTail() (string, error) {
......@@ -361,7 +361,7 @@ func (n *NEOGoSrv) masterAddrSlice() []string {
}
func (n *NEOGoSrv) URL() string {
return fmt.Sprintf("%s%s/%s", n.opt.URLPrefix(), strings.Join(n.masterAddrSlice(), ","), n.opt.name)
return fmt.Sprintf("%s%s%s", n.opt.URLPrefix(), strings.Join(n.masterAddrSlice(), ","), n.opt.URLQuery())
}
......@@ -391,22 +391,35 @@ func (opt NEOSrvOptions) Key() string {
}
// URLPrefix returns start of URL for a NEO server started with opt.
// e.g. neo:// or neos://ca=1;cert=2;key=3@
// To be come complete returned URL has to be appended with host and path parts.
// e.g. neo://test@ or neos://test@
// To become complete returned URL has to be appended with host and query parts.
func (opt NEOSrvOptions) URLPrefix() string {
zurl := ""
if !opt.SSL {
zurl = "neo://"
} else {
zurl = "neos://"
zurl += "ca=" + url.QueryEscape(opt.CA()) +";"
zurl += "cert=" + url.QueryEscape(opt.Cert()) +";"
zurl += "key=" + url.QueryEscape(opt.Key())
zurl += "@"
}
zurl += opt.name
zurl += "@"
return zurl
}
// URLQuery returns query part of the URL for a NEO server started with opt.
// e.g. ?ca=...&cert=...&key=...
func (opt NEOSrvOptions) URLQuery() string {
query := ""
if opt.SSL {
query += "ca=" + url.QueryEscape(opt.CA()) +"&"
query += "cert=" + url.QueryEscape(opt.Cert()) +"&"
query += "key=" + url.QueryEscape(opt.Key())
}
if query != "" {
query = "?" + query
}
return query
}
// ----------------
......@@ -677,14 +690,14 @@ func TestWatch(t *testing.T) {
// scheme neo(s)://[credentials@]master1,master2,...,masterN/name?options)
func TestParseURL(t *testing.T) {
// Most simple valid URI
testParseURL(t, "neo://127.0.0.1/test", urlInfo{})
testParseURL(t, "neo://test@127.0.0.1", urlInfo{})
// With 2 masters
testParseURL(t, "neo://127.0.0.1,127.0.0.2/test", urlInfo{masterAddr: "127.0.0.1,127.0.0.2"})
testParseURL(t, "neo://test@127.0.0.1,127.0.0.2", urlInfo{masterAddr: "127.0.0.1,127.0.0.2"})
// With ssl
u := "neos://ca=ca;cert=cert;key=key@127.0.0.1/test"
u := "neos://test@127.0.0.1?ca=ca&cert=cert&key=key"
testParseURL(t, u, urlInfo{netcfg: neonet.Config{CA: "ca", Cert: "cert", Key: "key"}})
// With query parameters
u = "neo://127.0.0.1/test?compress=true&logfile=n.log&cache-size=256"
u = "neo://test@127.0.0.1/compress=true&logfile=n.log&cache-size=256"
testParseURL(t, u, urlInfo{})
}
......
go/neo/t/neotest
View file @ af03f483
......@@ -499,7 +499,7 @@ GENsqlite() {
NEOpylite
# NOTE compression is disabled because when benchmarking server latency
# we do not want the time client(s) take to decompress data to interfere.
${dataset}_gen_data neo://$Mbind/$neocluster?compress=false $dataset_size
${dataset}_gen_data neo://$neocluster@$Mbind?compress=false $dataset_size
xneoctl set cluster stopping
wait # XXX fragile - won't work if there are children spawned outside
sync
......@@ -512,7 +512,7 @@ GENsql() {
echo -e '\n*** generating sql data...'
NEOpysql
# NOTE compression is disabled - see ^^^ (sqlite) for rationale.
${dataset}_gen_data neo://$Mbind/$neocluster?compress=false $dataset_size
${dataset}_gen_data neo://$neocluster@$Mbind?compress=false $dataset_size
xneoctl set cluster stopping
sleep 1 # XXX fragile
xmysql -e "SHUTDOWN"
......@@ -1194,21 +1194,21 @@ zbench_local() {
# XXX save time - we show only neo/py(!log)/sqlite
#echo -e "\n*** NEO/py sqlite"
#NEOpylite
#zbench neo://$Mbind/$neocluster neo/py/sqlite·P$Pneo $zhashok
#zbench neo://$neocluster@$Mbind neo/py/sqlite·P$Pneo $zhashok
#xneoctl set cluster stopping
#wait
# XXX JM asked to also have NEO/py with logging disabled
echo -e "\n*** NEO/py sqlite (logging disabled)"
X_NEOPY_LOG_SKIP=y NEOpylite
zbench neo://$Mbind/$neocluster "neo/py(!log)/sqlite·P$Pneo" $zhashok
zbench neo://$neocluster@$Mbind "neo/py(!log)/sqlite·P$Pneo" $zhashok
xneoctl set cluster stopping
wait
# XXX save time - we show only neo/py(!log)/sql
#echo -e "\n*** NEO/py sql"
#NEOpysql
#zbench neo://$Mbind/$neocluster neo/py/sql·P$Pneo $zhashok
#zbench neo://$neocluster@$Mbind neo/py/sql·P$Pneo $zhashok
#xneoctl set cluster stopping
#xmysql -e "SHUTDOWN"
#wait
......@@ -1216,27 +1216,27 @@ zbench_local() {
# XXX JM asked to also have NEO/py with logging disabled
echo -e "\n*** NEO/py sql (logging disabled)"
X_NEOPY_LOG_SKIP=y NEOpysql
zbench neo://$Mbind/$neocluster "neo/py(!log)/sql·P$Pneo" $zhashok
zbench neo://$neocluster@$Mbind "neo/py(!log)/sql·P$Pneo" $zhashok
xneoctl set cluster stopping
xmysql -e "SHUTDOWN"
wait
echo -e "\n*** NEO/go fs1"
NEOgofs1
zbench neo://$Mbind/$neocluster neo/go/fs1·P1 $zhashok
zbench neo://$neocluster@$Mbind neo/go/fs1·P1 $zhashok
xneoctl set cluster stopping
wait
echo -e "\n*** NEO/go fs1 (sha1 disabled on: storage, client)"
X_NEOGO_SHA1_SKIP=y NEOgofs1
X_NEOGO_SHA1_SKIP=y zbench_go neo://$Mbind/$neocluster "neo/go/fs1(!sha1)·P1" $zhashok
X_NEOGO_SHA1_SKIP=y zbench_go neo://$neocluster@$Mbind "neo/go/fs1(!sha1)·P1" $zhashok
xneoctl set cluster stopping
wait
echo -e "\n*** NEO/go sqlite"
if [ $Pneo == 1 ]; then
NEOgolite
zbench neo://$Mbind/$neocluster@ neo/go/sqlite·P$Pneo $zhashok
zbench neo://$neocluster@$Mbind neo/go/sqlite·P$Pneo $zhashok
xneoctl set cluster stopping
wait
else
......@@ -1246,7 +1246,7 @@ zbench_local() {
echo -e "\n*** NEO/go sqlite (sha1 disabled on: client)"
if [ $Pneo == 1 ]; then
NEOgolite
X_NEOGO_SHA1_SKIP=y zbench_go neo://$Mbind/$neocluster "neo/go/sqlite·P$Pneo" $zhashok
X_NEOGO_SHA1_SKIP=y zbench_go neo://$neocluster@$Mbind "neo/go/sqlite·P$Pneo" $zhashok
xneoctl set cluster stopping
wait
else
......@@ -1315,21 +1315,21 @@ zbench_cluster() {
# XXX save time - we show only neo/py(!log)/sqlite
#echo -e "\n*** NEO/py sqlite"
#NEOpylite
#on $url ./neotest zbench-client neo://$Mbind/$neocluster neo/py/sqlite·P$Pneo $zhashok
#on $url ./neotest zbench-client neo://$neocluster@$Mbind neo/py/sqlite·P$Pneo $zhashok
#xneoctl set cluster stopping
#wait
# XXX JM asked to also have NEO/py with logging disabled
echo -e "\n*** NEO/py sqlite (logging disabled)"
X_NEOPY_LOG_SKIP=y NEOpylite
on $url ./neotest zbench-client neo://$Mbind/$neocluster "\\\"neo/py(!log)/sqlite\\\"·P$Pneo" $zhashok
on $url ./neotest zbench-client neo://$neocluster@$Mbind "\\\"neo/py(!log)/sqlite\\\"·P$Pneo" $zhashok
xneoctl set cluster stopping
wait
# XXX save time - we show only neo/py(!log)/sql
#echo -e "\n*** NEO/py sql"
#NEOpysql
#on $url ./neotest zbench-client neo://$Mbind/$neocluster neo/py/sql·P$Pneo $zhashok
#on $url ./neotest zbench-client neo://$neocluster@$Mbind neo/py/sql·P$Pneo $zhashok
#xneoctl set cluster stopping
#xmysql -e "SHUTDOWN"
#wait
......@@ -1337,27 +1337,27 @@ zbench_cluster() {
# XXX JM asked to also have NEO/py with logging disabled
echo -e "\n*** NEO/py sql (logging disabled)"
X_NEOPY_LOG_SKIP=y NEOpysql
on $url ./neotest zbench-client neo://$Mbind/$neocluster "\\\"neo/py(!log)/sql\\\"·P$Pneo" $zhashok
on $url ./neotest zbench-client neo://$neocluster@$Mbind "\\\"neo/py(!log)/sql\\\"·P$Pneo" $zhashok
xneoctl set cluster stopping
xmysql -e "SHUTDOWN"
wait
echo -e "\n*** NEO/go fs"
NEOgofs1
on $url ./neotest zbench-client neo://$Mbind/$neocluster neo/go/fs1·P1 $zhashok
on $url ./neotest zbench-client neo://$neocluster@$Mbind neo/go/fs1·P1 $zhashok
xneoctl set cluster stopping
wait
echo -e "\n*** NEO/go fs1 (sha1 disabled on: storage, client)"
X_NEOGO_SHA1_SKIP=y NEOgofs1
on $url X_NEOGO_SHA1_SKIP=y ./neotest zbench-client --goonly neo://$Mbind/$neocluster "\\\"neo/go/fs1(!sha1)\\\"·P1" $zhashok
on $url X_NEOGO_SHA1_SKIP=y ./neotest zbench-client --goonly neo://$neocluster@$Mbind "\\\"neo/go/fs1(!sha1)\\\"·P1" $zhashok
xneoctl set cluster stopping
wait
echo -e "\n*** NEO/go sqlite"
if [ $Pneo == 1 ]; then
NEOgolite
on $url ./neotest zbench-client neo://$Mbind/$neocluster neo/go/sqlite·P$Pneo $zhashok
on $url ./neotest zbench-client neo://$neocluster@$Mbind neo/go/sqlite·P$Pneo $zhashok
xneoctl set cluster stopping
wait
else
......@@ -1367,7 +1367,7 @@ zbench_cluster() {
echo -e "\n*** NEO/go sqlite (sha1 disabled on: client)"
if [ $Pneo == 1 ]; then
NEOgolite
on $url X_NEOGO_SHA1_SKIP=y ./neotest zbench-client --goonly neo://$Mbind/$neocluster "\\\"neo/go/sqlite\\\"·P$Pneo" $zhashok
on $url X_NEOGO_SHA1_SKIP=y ./neotest zbench-client --goonly neo://$neocluster@$Mbind "\\\"neo/go/sqlite\\\"·P$Pneo" $zhashok
xneoctl set cluster stopping
wait
else
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7