Skip to content

N
neoppod
Commit af03f483 authored by Levin Zimmermann's avatar Levin Zimmermann
Browse files
Options

go/client/zurl: Sync format to py upstream 

NEO/go and NEO/py zurl format diverged over time:

- kirr/neo@8c974485

However with nexedi/neoppod!21 a
common solution was found. From there, this patch aims to adjust NEO/go zurl
format to be in sync with NEO/py zurl format again.
parent 1ad088c8
Show whitespace changes
Inline Side-by-side
Showing with 66 additions and 68 deletions
go/neo/client.go
View file @ af03f483
// Copyright (C) 2017-2023 Nexedi SA and Contributors. // Copyright (C) 2017-2024 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com> // Kirill Smelkov <kirr@nexedi.com>
// //
// This program is free software: you can Use, Study, Modify and Redistribute // This program is free software: you can Use, Study, Modify and Redistribute
...@@ -484,7 +484,7 @@ func openClientByURL(ctx context.Context, u *url.URL, opt *zodb.DriverOptions) ( ...@@ -484,7 +484,7 @@ func openClientByURL(ctx context.Context, u *url.URL, opt *zodb.DriverOptions) (
// parseURL extracts information from a NEO URI and puts this information into // parseURL extracts information from a NEO URI and puts this information into
// a urlInfo. // a urlInfo.
func parseURL(ctx context.Context, u *url.URL) (urlinfo *urlInfo, err error) { func parseURL(ctx context.Context, u *url.URL) (urlinfo *urlInfo, err error) {
// neo(s)://[credentials@]master1,master2,...,masterN/name?options // neo(s)://name@master1,master2,...,masterN?options
var ssl bool var ssl bool
switch u.Scheme { switch u.Scheme {
...@@ -493,50 +493,35 @@ func parseURL(ctx context.Context, u *url.URL) (urlinfo *urlInfo, err error) { ...@@ -493,50 +493,35 @@ func parseURL(ctx context.Context, u *url.URL) (urlinfo *urlInfo, err error) {
default: return nil, fmt.Errorf("invalid scheme") default: return nil, fmt.Errorf("invalid scheme")
} }
cred := u.User.String() name := u.User.String()
// ca=ca.crt;cert=my.crt;key=my.key if name == "" {
cred = strings.ReplaceAll(cred, ";", "&") // ; is no longer in default separators set https://github.com/golang/go/issues/25192 return nil, fmt.Errorf("cluster name not specified")
x, err := xurl.ParseQuery(cred) }
q, err := xurl.ParseQuery(u.RawQuery)
if err != nil { if err != nil {
return nil, fmt.Errorf("credentials: %s", err) return nil, err
} }
// xpop pops k from credentials, defaulting to $NEO_<K> if envok.
xpop := func(k string, envok bool) string { // qpop pops k from query, defaulting to $NEO_<K> if envok.
v, ok := x[k] qpop := func(k string, envok bool) string {
v, ok := q[k]
if !ok && envok { if !ok && envok {
v = os.Getenv("NEO_"+strings.ToUpper(k)) v = os.Getenv("NEO_"+strings.ToUpper(k))
} }
delete(x, k) delete(q, k)
return v return v
} }
netcfg := neonet.Config{} netcfg := neonet.Config{}
netcfg.LoNode = xpop("lonode", false) netcfg.LoNode = qpop("lonode", false)
if !ssl { // neos:// force TLS to be used and take ca/cert/key from environment if
if len(x) != 0 { // TLS credentials are not explicitly specified in uri
return nil, fmt.Errorf("credentials can be specified only with neos:// scheme") // neo:// use TLS only if ca/cert/key are explicitly specified in uri
} netcfg.CA = qpop("ca", ssl)
} else { netcfg.Cert = qpop("cert", ssl)
netcfg.CA = xpop("ca", true) netcfg.Key = qpop("key", ssl)
netcfg.Cert = xpop("cert", true)
netcfg.Key = xpop("key", true)
if len(x) != 0 {
return nil, fmt.Errorf("invalid credentials: %v", x)
}
}
name := u.Path
name = strings.TrimPrefix(name, "/")
if name == "" {
return nil, fmt.Errorf("cluster name not specified")
}
q, err := xurl.ParseQuery(u.RawQuery)
if err != nil {
return nil, err
}
// pop not yet used client options // pop not yet used client options
// (our neo client doesn't apply their effect yet) // (our neo client doesn't apply their effect yet)
......
go/neo/client_test.go
View file @ af03f483
// Copyright (C) 2020-2023 Nexedi SA and Contributors. // Copyright (C) 2020-2024 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com> // Kirill Smelkov <kirr@nexedi.com>
// //
// This program is free software: you can Use, Study, Modify and Redistribute // This program is free software: you can Use, Study, Modify and Redistribute
...@@ -175,7 +175,7 @@ func (n *NEOPySrv) clusterName() string { ...@@ -175,7 +175,7 @@ func (n *NEOPySrv) clusterName() string {
} }
func (n *NEOPySrv) URL() string { func (n *NEOPySrv) URL() string {
return fmt.Sprintf("%s%s/%s", n.opt.URLPrefix(), strings.Join(n.masterAddrSlice, ","), n.clusterName()) return fmt.Sprintf("%s%s%s", n.opt.URLPrefix(), strings.Join(n.masterAddrSlice, ","), n.opt.URLQuery())
} }
func (n *NEOPySrv) LogTail() (string, error) { func (n *NEOPySrv) LogTail() (string, error) {
...@@ -361,7 +361,7 @@ func (n *NEOGoSrv) masterAddrSlice() []string { ...@@ -361,7 +361,7 @@ func (n *NEOGoSrv) masterAddrSlice() []string {
} }
func (n *NEOGoSrv) URL() string { func (n *NEOGoSrv) URL() string {
return fmt.Sprintf("%s%s/%s", n.opt.URLPrefix(), strings.Join(n.masterAddrSlice(), ","), n.opt.name) return fmt.Sprintf("%s%s%s", n.opt.URLPrefix(), strings.Join(n.masterAddrSlice(), ","), n.opt.URLQuery())
} }
...@@ -391,22 +391,35 @@ func (opt NEOSrvOptions) Key() string { ...@@ -391,22 +391,35 @@ func (opt NEOSrvOptions) Key() string {
} }
// URLPrefix returns start of URL for a NEO server started with opt. // URLPrefix returns start of URL for a NEO server started with opt.
// e.g. neo:// or neos://ca=1;cert=2;key=3@ // e.g. neo://test@ or neos://test@
// To be come complete returned URL has to be appended with host and path parts. // To become complete returned URL has to be appended with host and query parts.
func (opt NEOSrvOptions) URLPrefix() string { func (opt NEOSrvOptions) URLPrefix() string {
zurl := "" zurl := ""
if !opt.SSL { if !opt.SSL {
zurl = "neo://" zurl = "neo://"
} else { } else {
zurl = "neos://" zurl = "neos://"
zurl += "ca=" + url.QueryEscape(opt.CA()) +";"
zurl += "cert=" + url.QueryEscape(opt.Cert()) +";"
zurl += "key=" + url.QueryEscape(opt.Key())
zurl += "@"
} }
zurl += opt.name
zurl += "@"
return zurl return zurl
} }
// URLQuery returns query part of the URL for a NEO server started with opt.
// e.g. ?ca=...&cert=...&key=...
func (opt NEOSrvOptions) URLQuery() string {
query := ""
if opt.SSL {
query += "ca=" + url.QueryEscape(opt.CA()) +"&"
query += "cert=" + url.QueryEscape(opt.Cert()) +"&"
query += "key=" + url.QueryEscape(opt.Key())
}
if query != "" {
query = "?" + query
}
return query
}
// ---------------- // ----------------
...@@ -677,14 +690,14 @@ func TestWatch(t *testing.T) { ...@@ -677,14 +690,14 @@ func TestWatch(t *testing.T) {
// scheme neo(s)://[credentials@]master1,master2,...,masterN/name?options) // scheme neo(s)://[credentials@]master1,master2,...,masterN/name?options)
func TestParseURL(t *testing.T) { func TestParseURL(t *testing.T) {
// Most simple valid URI // Most simple valid URI
testParseURL(t, "neo://127.0.0.1/test", urlInfo{}) testParseURL(t, "neo://test@127.0.0.1", urlInfo{})
// With 2 masters // With 2 masters
testParseURL(t, "neo://127.0.0.1,127.0.0.2/test", urlInfo{masterAddr: "127.0.0.1,127.0.0.2"}) testParseURL(t, "neo://test@127.0.0.1,127.0.0.2", urlInfo{masterAddr: "127.0.0.1,127.0.0.2"})
// With ssl // With ssl
u := "neos://ca=ca;cert=cert;key=key@127.0.0.1/test" u := "neos://test@127.0.0.1?ca=ca&cert=cert&key=key"
testParseURL(t, u, urlInfo{netcfg: neonet.Config{CA: "ca", Cert: "cert", Key: "key"}}) testParseURL(t, u, urlInfo{netcfg: neonet.Config{CA: "ca", Cert: "cert", Key: "key"}})
// With query parameters // With query parameters
u = "neo://127.0.0.1/test?compress=true&logfile=n.log&cache-size=256" u = "neo://test@127.0.0.1/compress=true&logfile=n.log&cache-size=256"
testParseURL(t, u, urlInfo{}) testParseURL(t, u, urlInfo{})
} }
......
go/neo/t/neotest
View file @ af03f483
...@@ -499,7 +499,7 @@ GENsqlite() { ...@@ -499,7 +499,7 @@ GENsqlite() {
NEOpylite NEOpylite
# NOTE compression is disabled because when benchmarking server latency # NOTE compression is disabled because when benchmarking server latency
# we do not want the time client(s) take to decompress data to interfere. # we do not want the time client(s) take to decompress data to interfere.
${dataset}_gen_data neo://$Mbind/$neocluster?compress=false $dataset_size ${dataset}_gen_data neo://$neocluster@$Mbind?compress=false $dataset_size
xneoctl set cluster stopping xneoctl set cluster stopping
wait # XXX fragile - won't work if there are children spawned outside wait # XXX fragile - won't work if there are children spawned outside
sync sync
...@@ -512,7 +512,7 @@ GENsql() { ...@@ -512,7 +512,7 @@ GENsql() {
echo -e '\n*** generating sql data...' echo -e '\n*** generating sql data...'
NEOpysql NEOpysql
# NOTE compression is disabled - see ^^^ (sqlite) for rationale. # NOTE compression is disabled - see ^^^ (sqlite) for rationale.
${dataset}_gen_data neo://$Mbind/$neocluster?compress=false $dataset_size ${dataset}_gen_data neo://$neocluster@$Mbind?compress=false $dataset_size
xneoctl set cluster stopping xneoctl set cluster stopping
sleep 1 # XXX fragile sleep 1 # XXX fragile
xmysql -e "SHUTDOWN" xmysql -e "SHUTDOWN"
...@@ -1194,21 +1194,21 @@ zbench_local() { ...@@ -1194,21 +1194,21 @@ zbench_local() {
# XXX save time - we show only neo/py(!log)/sqlite # XXX save time - we show only neo/py(!log)/sqlite
#echo -e "\n*** NEO/py sqlite" #echo -e "\n*** NEO/py sqlite"
#NEOpylite #NEOpylite
#zbench neo://$Mbind/$neocluster neo/py/sqlite·P$Pneo $zhashok #zbench neo://$neocluster@$Mbind neo/py/sqlite·P$Pneo $zhashok
#xneoctl set cluster stopping #xneoctl set cluster stopping
#wait #wait
# XXX JM asked to also have NEO/py with logging disabled # XXX JM asked to also have NEO/py with logging disabled
echo -e "\n*** NEO/py sqlite (logging disabled)" echo -e "\n*** NEO/py sqlite (logging disabled)"
X_NEOPY_LOG_SKIP=y NEOpylite X_NEOPY_LOG_SKIP=y NEOpylite
zbench neo://$Mbind/$neocluster "neo/py(!log)/sqlite·P$Pneo" $zhashok zbench neo://$neocluster@$Mbind "neo/py(!log)/sqlite·P$Pneo" $zhashok
xneoctl set cluster stopping xneoctl set cluster stopping
wait wait
# XXX save time - we show only neo/py(!log)/sql # XXX save time - we show only neo/py(!log)/sql
#echo -e "\n*** NEO/py sql" #echo -e "\n*** NEO/py sql"
#NEOpysql #NEOpysql
#zbench neo://$Mbind/$neocluster neo/py/sql·P$Pneo $zhashok #zbench neo://$neocluster@$Mbind neo/py/sql·P$Pneo $zhashok
#xneoctl set cluster stopping #xneoctl set cluster stopping
#xmysql -e "SHUTDOWN" #xmysql -e "SHUTDOWN"
#wait #wait
...@@ -1216,27 +1216,27 @@ zbench_local() { ...@@ -1216,27 +1216,27 @@ zbench_local() {
# XXX JM asked to also have NEO/py with logging disabled # XXX JM asked to also have NEO/py with logging disabled
echo -e "\n*** NEO/py sql (logging disabled)" echo -e "\n*** NEO/py sql (logging disabled)"
X_NEOPY_LOG_SKIP=y NEOpysql X_NEOPY_LOG_SKIP=y NEOpysql
zbench neo://$Mbind/$neocluster "neo/py(!log)/sql·P$Pneo" $zhashok zbench neo://$neocluster@$Mbind "neo/py(!log)/sql·P$Pneo" $zhashok
xneoctl set cluster stopping xneoctl set cluster stopping
xmysql -e "SHUTDOWN" xmysql -e "SHUTDOWN"
wait wait
echo -e "\n*** NEO/go fs1" echo -e "\n*** NEO/go fs1"
NEOgofs1 NEOgofs1
zbench neo://$Mbind/$neocluster neo/go/fs1·P1 $zhashok zbench neo://$neocluster@$Mbind neo/go/fs1·P1 $zhashok
xneoctl set cluster stopping xneoctl set cluster stopping
wait wait
echo -e "\n*** NEO/go fs1 (sha1 disabled on: storage, client)" echo -e "\n*** NEO/go fs1 (sha1 disabled on: storage, client)"
X_NEOGO_SHA1_SKIP=y NEOgofs1 X_NEOGO_SHA1_SKIP=y NEOgofs1
X_NEOGO_SHA1_SKIP=y zbench_go neo://$Mbind/$neocluster "neo/go/fs1(!sha1)·P1" $zhashok X_NEOGO_SHA1_SKIP=y zbench_go neo://$neocluster@$Mbind "neo/go/fs1(!sha1)·P1" $zhashok
xneoctl set cluster stopping xneoctl set cluster stopping
wait wait
echo -e "\n*** NEO/go sqlite" echo -e "\n*** NEO/go sqlite"
if [ $Pneo == 1 ]; then if [ $Pneo == 1 ]; then
NEOgolite NEOgolite
zbench neo://$Mbind/$neocluster@ neo/go/sqlite·P$Pneo $zhashok zbench neo://$neocluster@$Mbind neo/go/sqlite·P$Pneo $zhashok
xneoctl set cluster stopping xneoctl set cluster stopping
wait wait
else else
...@@ -1246,7 +1246,7 @@ zbench_local() { ...@@ -1246,7 +1246,7 @@ zbench_local() {
echo -e "\n*** NEO/go sqlite (sha1 disabled on: client)" echo -e "\n*** NEO/go sqlite (sha1 disabled on: client)"
if [ $Pneo == 1 ]; then if [ $Pneo == 1 ]; then
NEOgolite NEOgolite
X_NEOGO_SHA1_SKIP=y zbench_go neo://$Mbind/$neocluster "neo/go/sqlite·P$Pneo" $zhashok X_NEOGO_SHA1_SKIP=y zbench_go neo://$neocluster@$Mbind "neo/go/sqlite·P$Pneo" $zhashok
xneoctl set cluster stopping xneoctl set cluster stopping
wait wait
else else
...@@ -1315,21 +1315,21 @@ zbench_cluster() { ...@@ -1315,21 +1315,21 @@ zbench_cluster() {
# XXX save time - we show only neo/py(!log)/sqlite # XXX save time - we show only neo/py(!log)/sqlite
#echo -e "\n*** NEO/py sqlite" #echo -e "\n*** NEO/py sqlite"
#NEOpylite #NEOpylite
#on $url ./neotest zbench-client neo://$Mbind/$neocluster neo/py/sqlite·P$Pneo $zhashok #on $url ./neotest zbench-client neo://$neocluster@$Mbind neo/py/sqlite·P$Pneo $zhashok
#xneoctl set cluster stopping #xneoctl set cluster stopping
#wait #wait
# XXX JM asked to also have NEO/py with logging disabled # XXX JM asked to also have NEO/py with logging disabled
echo -e "\n*** NEO/py sqlite (logging disabled)" echo -e "\n*** NEO/py sqlite (logging disabled)"
X_NEOPY_LOG_SKIP=y NEOpylite X_NEOPY_LOG_SKIP=y NEOpylite
on $url ./neotest zbench-client neo://$Mbind/$neocluster "\\\"neo/py(!log)/sqlite\\\"·P$Pneo" $zhashok on $url ./neotest zbench-client neo://$neocluster@$Mbind "\\\"neo/py(!log)/sqlite\\\"·P$Pneo" $zhashok
xneoctl set cluster stopping xneoctl set cluster stopping
wait wait
# XXX save time - we show only neo/py(!log)/sql # XXX save time - we show only neo/py(!log)/sql
#echo -e "\n*** NEO/py sql" #echo -e "\n*** NEO/py sql"
#NEOpysql #NEOpysql
#on $url ./neotest zbench-client neo://$Mbind/$neocluster neo/py/sql·P$Pneo $zhashok #on $url ./neotest zbench-client neo://$neocluster@$Mbind neo/py/sql·P$Pneo $zhashok
#xneoctl set cluster stopping #xneoctl set cluster stopping
#xmysql -e "SHUTDOWN" #xmysql -e "SHUTDOWN"
#wait #wait
...@@ -1337,27 +1337,27 @@ zbench_cluster() { ...@@ -1337,27 +1337,27 @@ zbench_cluster() {
# XXX JM asked to also have NEO/py with logging disabled # XXX JM asked to also have NEO/py with logging disabled
echo -e "\n*** NEO/py sql (logging disabled)" echo -e "\n*** NEO/py sql (logging disabled)"
X_NEOPY_LOG_SKIP=y NEOpysql X_NEOPY_LOG_SKIP=y NEOpysql
on $url ./neotest zbench-client neo://$Mbind/$neocluster "\\\"neo/py(!log)/sql\\\"·P$Pneo" $zhashok on $url ./neotest zbench-client neo://$neocluster@$Mbind "\\\"neo/py(!log)/sql\\\"·P$Pneo" $zhashok
xneoctl set cluster stopping xneoctl set cluster stopping
xmysql -e "SHUTDOWN" xmysql -e "SHUTDOWN"
wait wait
echo -e "\n*** NEO/go fs" echo -e "\n*** NEO/go fs"
NEOgofs1 NEOgofs1
on $url ./neotest zbench-client neo://$Mbind/$neocluster neo/go/fs1·P1 $zhashok on $url ./neotest zbench-client neo://$neocluster@$Mbind neo/go/fs1·P1 $zhashok
xneoctl set cluster stopping xneoctl set cluster stopping
wait wait
echo -e "\n*** NEO/go fs1 (sha1 disabled on: storage, client)" echo -e "\n*** NEO/go fs1 (sha1 disabled on: storage, client)"
X_NEOGO_SHA1_SKIP=y NEOgofs1 X_NEOGO_SHA1_SKIP=y NEOgofs1
on $url X_NEOGO_SHA1_SKIP=y ./neotest zbench-client --goonly neo://$Mbind/$neocluster "\\\"neo/go/fs1(!sha1)\\\"·P1" $zhashok on $url X_NEOGO_SHA1_SKIP=y ./neotest zbench-client --goonly neo://$neocluster@$Mbind "\\\"neo/go/fs1(!sha1)\\\"·P1" $zhashok
xneoctl set cluster stopping xneoctl set cluster stopping
wait wait
echo -e "\n*** NEO/go sqlite" echo -e "\n*** NEO/go sqlite"
if [ $Pneo == 1 ]; then if [ $Pneo == 1 ]; then
NEOgolite NEOgolite
on $url ./neotest zbench-client neo://$Mbind/$neocluster neo/go/sqlite·P$Pneo $zhashok on $url ./neotest zbench-client neo://$neocluster@$Mbind neo/go/sqlite·P$Pneo $zhashok
xneoctl set cluster stopping xneoctl set cluster stopping
wait wait
else else
...@@ -1367,7 +1367,7 @@ zbench_cluster() { ...@@ -1367,7 +1367,7 @@ zbench_cluster() {
echo -e "\n*** NEO/go sqlite (sha1 disabled on: client)" echo -e "\n*** NEO/go sqlite (sha1 disabled on: client)"
if [ $Pneo == 1 ]; then if [ $Pneo == 1 ]; then
NEOgolite NEOgolite
on $url X_NEOGO_SHA1_SKIP=y ./neotest zbench-client --goonly neo://$Mbind/$neocluster "\\\"neo/go/sqlite\\\"·P$Pneo" $zhashok on $url X_NEOGO_SHA1_SKIP=y ./neotest zbench-client --goonly neo://$neocluster@$Mbind "\\\"neo/go/sqlite\\\"·P$Pneo" $zhashok
xneoctl set cluster stopping xneoctl set cluster stopping
wait wait
else else
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7