Commit 354887e3 authored by Kirill Smelkov's avatar Kirill Smelkov

fixup! Node: Add support for NEO cluster with > 1 master

- no need to keep Node.MasterAddr anymore - the address of current PM is
  managed by TalkMaster and is provided as part of operational context
  to user functions that TalkMaster runs.

- correct docstrings.

- cosmetics.
parent 20f76288
// Copyright (C) 2017-2021 Nexedi SA and Contributors. // Copyright (C) 2017-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com> // Kirill Smelkov <kirr@nexedi.com>
// //
// This program is free software: you can Use, Study, Modify and Redistribute // This program is free software: you can Use, Study, Modify and Redistribute
...@@ -78,7 +78,7 @@ var _ zodb.IStorageDriver = (*Client)(nil) ...@@ -78,7 +78,7 @@ var _ zodb.IStorageDriver = (*Client)(nil)
// NewClient creates new client node. // NewClient creates new client node.
// //
// It will connect to master @masterAddr and identify with specified cluster name. // It will connect to a master with address from masterAddrSlice and identify with specified cluster name.
// Use Run to actually start running the node. // Use Run to actually start running the node.
func NewClient(clusterName string, masterAddrSlice []string, net xnet.Networker) *Client { func NewClient(clusterName string, masterAddrSlice []string, net xnet.Networker) *Client {
c := &Client{ c := &Client{
...@@ -542,8 +542,7 @@ func (c *Client) URL() string { ...@@ -542,8 +542,7 @@ func (c *Client) URL() string {
if strings.Contains(c.node.Net.Network(), "+tls") { if strings.Contains(c.node.Net.Network(), "+tls") {
zurl += "s" zurl += "s"
} }
zurl += fmt.Sprintf( zurl += fmt.Sprintf("://%s/%s", strings.Join(c.node.MasterAddrSlice, ","), c.node.ClusterName,
"://%s/%s", strings.Join(c.node.MasterAddrSlice, ","), c.node.ClusterName,
) )
return zurl return zurl
} }
......
...@@ -219,8 +219,7 @@ func (m *Master) Run(ctx context.Context, l xnet.Listener) (err error) { ...@@ -219,8 +219,7 @@ func (m *Master) Run(ctx context.Context, l xnet.Listener) (err error) {
if err != nil { if err != nil {
return err return err
} }
// NOTE: How can this master node know the address of the // XXX How can this master node know the address of the other master nodes?
// other master nodes?
m.node.MasterAddrSlice = []string{addr.String()} m.node.MasterAddrSlice = []string{addr.String()}
m.node.MyInfo = proto.NodeInfo{ m.node.MyInfo = proto.NodeInfo{
Type: proto.MASTER, Type: proto.MASTER,
......
// Copyright (C) 2017-2021 Nexedi SA and Contributors. // Copyright (C) 2017-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com> // Kirill Smelkov <kirr@nexedi.com>
// //
// This program is free software: you can Use, Study, Modify and Redistribute // This program is free software: you can Use, Study, Modify and Redistribute
...@@ -84,7 +84,9 @@ const ( ...@@ -84,7 +84,9 @@ const (
δPartTabPassThrough _MasteredNodeFlags = 1 << iota δPartTabPassThrough _MasteredNodeFlags = 1 << iota
) )
// newMasteredNode creates new _MasteredNode that connects to masterAddr/cluster via net. // newMasteredNode creates new _MasteredNode that connects to masters/cluster via net.
//
// Addresses of known masters are specified by masterAddrSlice.
func newMasteredNode(typ proto.NodeType, clusterName string, net xnet.Networker, masterAddrSlice []string) *_MasteredNode { func newMasteredNode(typ proto.NodeType, clusterName string, net xnet.Networker, masterAddrSlice []string) *_MasteredNode {
node := &_MasteredNode{ node := &_MasteredNode{
Node: xneo.NewNode(typ, clusterName, net, masterAddrSlice), Node: xneo.NewNode(typ, clusterName, net, masterAddrSlice),
...@@ -110,22 +112,17 @@ func (node *_MasteredNode) TalkMaster(ctx context.Context, f func(context.Contex ...@@ -110,22 +112,17 @@ func (node *_MasteredNode) TalkMaster(ctx context.Context, f func(context.Contex
ctx0 := ctx ctx0 := ctx
// When a node is created with "NewNode", we don't know yet, which of the // When a node is created with "NewNode", we don't know yet, which of the
// the provided master nodes is the primary master. We'll figure this our here // the provided master nodes is the primary master. We'll figure this out here
// and simply start with the first node. // and simply start with the first node.
// We don't assign 'MasterAddr' in 'NewNode', because master nodes are also maddr := node.MasterAddrSlice[0]
// created with the 'NewNode' method and master nodes don't know their Addr yet
// during initialization time: it's only assigned later when 'Run' is called.
if (node.MasterAddr == "") {
node.MasterAddr = node.MasterAddrSlice[0]
}
defer task.Runningf(&ctx, "%s: talk master(%s)", node.MyInfo.NID, node.MasterAddr)(&err) defer task.Runningf(&ctx, "%s: talk master(%s)", node.MyInfo.NID, maddr)(&err)
for { for {
node.updateOperational(func() { node.updateOperational(func() {
node.mlink = nil node.mlink = nil
}) })
err := node.talkMaster1(ctx, ctx0, f, node.MasterAddr) err := node.talkMaster1(ctx, ctx0, maddr, f)
log.Warning(ctx, err) // XXX Warning -> Error? log.Warning(ctx, err) // XXX Warning -> Error?
if errors.Is(err, cmdShutdown) { if errors.Is(err, cmdShutdown) {
return err // M commands to shutdown return err // M commands to shutdown
...@@ -134,7 +131,7 @@ func (node *_MasteredNode) TalkMaster(ctx context.Context, f func(context.Contex ...@@ -134,7 +131,7 @@ func (node *_MasteredNode) TalkMaster(ctx context.Context, f func(context.Contex
var notPrimaryMaster *proto.NotPrimaryMaster var notPrimaryMaster *proto.NotPrimaryMaster
if errors.As(err, &notPrimaryMaster) { if errors.As(err, &notPrimaryMaster) {
primary := notPrimaryMaster.KnownMasterList[notPrimaryMaster.Primary] primary := notPrimaryMaster.KnownMasterList[notPrimaryMaster.Primary]
node.MasterAddr = fmt.Sprintf("[%s]:%v", primary.Host, primary.Port) maddr = fmt.Sprintf("[%s]:%v", primary.Host, primary.Port)
} }
// TODO if err == "reject identification / protocol error" -> shutdown client? // TODO if err == "reject identification / protocol error" -> shutdown client?
...@@ -151,7 +148,7 @@ func (node *_MasteredNode) TalkMaster(ctx context.Context, f func(context.Contex ...@@ -151,7 +148,7 @@ func (node *_MasteredNode) TalkMaster(ctx context.Context, f func(context.Contex
} }
} }
func (node *_MasteredNode) talkMaster1(ctx, ctxPreTalkM context.Context, f func(context.Context, *_MasterLink) error, masterAddr string) error { func (node *_MasteredNode) talkMaster1(ctx, ctxPreTalkM context.Context, maddr string, f func(context.Context, *_MasterLink) error) error {
reqID := &proto.RequestIdentification{ reqID := &proto.RequestIdentification{
NodeType: node.MyInfo.Type, NodeType: node.MyInfo.Type,
NID: node.MyInfo.NID, NID: node.MyInfo.NID,
...@@ -161,7 +158,7 @@ func (node *_MasteredNode) talkMaster1(ctx, ctxPreTalkM context.Context, f func( ...@@ -161,7 +158,7 @@ func (node *_MasteredNode) talkMaster1(ctx, ctxPreTalkM context.Context, f func(
DevPath: nil, // XXX stub DevPath: nil, // XXX stub
NewNID: nil, // XXX stub NewNID: nil, // XXX stub
} }
mlink, accept, err := xneo.Dial(ctx, proto.MASTER, node.Net, masterAddr, reqID) mlink, accept, err := xneo.Dial(ctx, proto.MASTER, node.Net, maddr, reqID)
if err != nil { if err != nil {
return err return err
} }
......
...@@ -79,8 +79,7 @@ type Node struct { ...@@ -79,8 +79,7 @@ type Node struct {
MyInfo proto.NodeInfo // type, laddr, nid, state, idtime MyInfo proto.NodeInfo // type, laddr, nid, state, idtime
ClusterName string ClusterName string
Net xnet.Networker // network AP we are sending/receiving on Net xnet.Networker // network AP we are sending/receiving on
MasterAddr string // address of current master TODO -> masterRegistry MasterAddrSlice []string // address of all known masters TODO -> masterRegistry
MasterAddrSlice []string // address of all known masters
// XXX reconsider not using State and have just .NodeTab, .PartTab, .ClusterState // XXX reconsider not using State and have just .NodeTab, .PartTab, .ClusterState
// StateMu sync.RWMutex // <- XXX unexport ? XXX not used -> move to MasteredNode ? // StateMu sync.RWMutex // <- XXX unexport ? XXX not used -> move to MasteredNode ?
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment