// Copyright (C) 2017-2021  Nexedi SA and Contributors.
//                          Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.

package neo
// master node

// Master organization
//
// Master is the node that oversees and manages how whole NEO cluster works.
// It accepts incoming connections, assigns node ID to incoming peers,
// maintains node and partition tables and broadcasts updates of those tables to
// all connected peers. It drives cluster through recovery -> verify -> serve
// cycle. It also manages commits initiated by clients and notifies clients
// about changes committed by another client (TODO).
//
// Master is organized as follows:
//
// - main task that controls whole logic of master working. It spawns subtasks
//   to implement that logic and communicates with the subtasks via channels.
//   Main is the only mutator of nodeTab, partTab, etc.
//
// - listen task that accepts incoming connections and hands them over to main
//   via nodeComeq.
//
// - per peer workers are spawned and interact with main via channels.
//   A worker might belong to a particular cluster state - e.g. storCtlRecovery
//   for recovery. In such cases worker lifetime is inside the lifetime of
//   corresponding cluster state. If a worker does not belong to a particular
//   cluster state - e.g. accept+notify - it overlives the state in which the
//   worker was started.
//
// - δnodeTab, δpartTab updates are proxied to peer by per-peer "notify" task.

import (
	"context"
	"errors"
	"fmt"
	"time"

	"lab.nexedi.com/kirr/go123/xcontext"
	"lab.nexedi.com/kirr/go123/xerr"
	"lab.nexedi.com/kirr/go123/xnet"
	"lab.nexedi.com/kirr/go123/xsync"

	"lab.nexedi.com/kirr/neo/go/neo/internal/xtime"
	"lab.nexedi.com/kirr/neo/go/neo/neonet"
	"lab.nexedi.com/kirr/neo/go/neo/proto"
	"lab.nexedi.com/kirr/neo/go/neo/xneo"
	"lab.nexedi.com/kirr/neo/go/zodb"
	"lab.nexedi.com/kirr/neo/go/internal/log"
	"lab.nexedi.com/kirr/neo/go/internal/task"
	xxcontext "lab.nexedi.com/kirr/neo/go/internal/xcontext"
	"lab.nexedi.com/kirr/neo/go/internal/xio"
)

// Master is a node overseeing and managing how whole NEO cluster works.
type Master struct {
	node *xneo.Node

	// whole Run runs under runCtx
	runCtx context.Context

	// "global" workgroup under which main, accept and tasks, that should
	// last for whole Run time, are spawned.
	mainWG *xsync.WorkGroup

	// main <- node come or go
	nodeComeq  chan nodeCome  // main <- accept "node connected"
	nodeLeaveq chan nodeLeave // main <- peer.wait "node (should be) disconnected"

	// in addition to nodeTab (which keeps information about a node) tasks
	// and mastering context that are specific to a peer are organized
	// around peerTab[peer.nid].
	peerTab map[proto.NodeID]*_MasteredPeer

	// last allocated oid & tid
	// XXX how to start allocating oid from 0, not 1 ?
	// TODO mu
	lastOid zodb.Oid
	lastTid zodb.Tid

	// channels controlling main driver
	ctlStart    chan chan error	// request to start cluster
	ctlStop     chan chan struct{}	// request to stop  cluster

	// so tests could override
	monotime func() float64
}

// nodeCome represents "node connects" event.
type nodeCome struct {
	req    *neonet.Request
	idReq  *proto.RequestIdentification // we received this identification request
}

// nodeLeave represents "node (should be) disconnected" event.
type nodeLeave struct {
	peer *_MasteredPeer
	err  error          // disconnect due to err
}

// _MasteredPeer represents context for all tasks related to one peer driven by master.
type _MasteredPeer struct {
	node *xneo.PeerNode

	// all tasks are spawned under wg. If any task fails - whole wg is canceled.
	// When wg is done, main is signalled that "peer (should be) disconnected".
	// cancel signals all tasks under wg to stop.
	wg     *xsync.WorkGroup
	cancel func()

	idReq     *neonet.Request             // peer's original identification request
	acceptMsg *proto.AcceptIdentification // how we decided to accept it

	// snapshot of nodeTab/partTab/stateCode when peer was accepted by main.
	state0 *xneo.ClusterStateSnapshot
	// main -> peer.notify  δnodeTab/δpartTab/δstateCode.
	notifyq chan _ΔClusterState
	// main -> peer.notify  "you are slow to consume updates"
	notifyqOverflow chan struct{}

	acceptSent chan struct{} // ready after initial accept sequence is sent to the peer
}

// _ΔClusterState represents δnodeTab/δpartTab/δClusterState.
type _ΔClusterState interface { δClusterState() }
type _ΔNodeTab struct {
	proto.NodeInfo // new value for change of 1 nodeTab entry
}
type _ΔPartTab struct {
	// TODO
}
type _ΔStateCode struct {
	proto.ClusterState // new value
}

func (_ *_ΔNodeTab)   δClusterState() {}
func (_ *_ΔPartTab)   δClusterState() {}
func (_ *_ΔStateCode) δClusterState() {}


// NewMaster creates new master node.
//
// Use Run to actually start running the node.
func NewMaster(clusterName string, net xnet.Networker) *Master {
	return &Master{
		node: xneo.NewNode(proto.MASTER, clusterName, net, ""),

		ctlStart:   make(chan chan error),
		ctlStop:    make(chan chan struct{}),
		nodeComeq:  make(chan nodeCome),
		nodeLeaveq: make(chan nodeLeave),
		peerTab:    make(map[proto.NodeID]*_MasteredPeer),
		monotime:   xtime.Mono,
	}
}

var ErrStartNonOperational = errors.New("start: cluster is non-operational")

// Start requests cluster to eventually transition into running state.
//
// It returns an error if such transition is not currently possible to begin
// (e.g. partition table is not operational).
// It returns nil if the transition began.
//
// NOTE upon successful return cluster is not yet in running state - the transition will
// take time and could be also automatically aborted due to cluster environment change (e.g.
// a storage node goes down).
func (m *Master) Start() error {
	ech := make(chan error)
	m.ctlStart <- ech
	return <-ech
}

// Stop requests cluster to eventually transition into recovery state.
func (m *Master) Stop()  {
	ech := make(chan struct{})
	m.ctlStop <- ech
	<-ech
}

// setClusterState sets .clusterState and notifies subscribers.
func (m *Master) setClusterState(ctx context.Context, state proto.ClusterState) {
	log.Infof(ctx, "cluster state <- %s", state)
	m.node.State.Code.Set(state)
	//m.notifyAll(ctx, &_ΔStateCode{state})		TODO enable
}


// Run starts master node and runs it until ctx is cancelled or fatal error.
//
// The master will be serving incoming connections on l.
func (m *Master) Run(ctx context.Context, l xnet.Listener) (err error) {
	mynid := m.allocNID(proto.MASTER)
	addr := l.Addr()
	defer task.Runningf(&ctx, "%s", mynid)(&err)
	log.Infof(ctx, "listening on %s ...", addr)
	m.runCtx = ctx
	m.mainWG = xsync.NewWorkGroup(m.runCtx)

	// update our master & serving address in node
	naddr, err := proto.Addr(addr)
	if err != nil {
		return err
	}
	m.node.MasterAddr = addr.String()
	m.node.MyInfo = proto.NodeInfo{
		Type:	proto.MASTER,
		Addr:	naddr,
		NID:	mynid,
		State:	proto.RUNNING,
		IdTime:	proto.IdTimeNone,	// XXX ok?
	}

	// update nodeTab with self
	m.updateNodeTab(ctx, m.node.MyInfo)

	// wrap listener with link / identification hello checker
	lli := xneo.NewListener(neonet.NewLinkListener(l))
	defer func() {
		__ := lli.Close()
		err = xerr.First(err, __)
	}()

	// listen: accept incoming connections and pass them to main driver
	m.mainWG.Go(func(ctx context.Context) (err error) {
		defer task.Running(&ctx, "listen")(&err)

		for {
			if ctx.Err() != nil {
				return ctx.Err()
			}

			req, idReq, err := lli.Accept(ctx)
			if err != nil {
				if !xxcontext.Canceled(err) {
					log.Error(ctx, err)	// XXX throttle?
				}
				continue
			}

			// for storages the only incoming connection is for RequestIdentification
			// and then master only drives it. So close accept as noone will be
			// listening for it on our side anymore.
			switch idReq.NodeType {
			case proto.CLIENT:
				// ok

			case proto.STORAGE:
				fallthrough
			default:
				req.Link().CloseAccept()
			}

			// handover to main driver
			select {
			case m.nodeComeq <- nodeCome{req, idReq}:
				// ok

			case <-ctx.Done():
				// shutdown
				xio.LClose(ctx, req.Link())
				continue
			}
		}
	})

	// main: main driving logic
	m.mainWG.Go(m.main)

	err = m.mainWG.Wait()
	// change `... canceled` to just canceled?
	// (e.g. `master: listen: canceled` or `master: main: canceled` -> `master: canceled`)
	if ctx.Err() != nil {
		err = ctx.Err()
	}

	return err
}

// main is the process that implements main master cluster management logic: node tracking, cluster
// state updates, scheduling data movement between storage nodes, etc.
//
// NOTE main's goroutine is the only mutator of nodeTab, partTab and other cluster state
func (m *Master) main(ctx context.Context) (err error) {
	defer task.Running(&ctx, "main")(&err)

	for ctx.Err() == nil {
		// recover partition table from storages and wait till enough
		// storages connects us so that we can see the partition table
		// can be operational.
		//
		// Successful recovery means all ^^^ preconditions are met and
		// a command came to us to start the cluster.
		err := m.recovery(ctx)
		if err != nil {
			// do not log - the error is returned up
			return err // recovery cancelled
		}
		log.Infof(ctx, "recovered ok; partTab:\n%s", m.node.State.PartTab)

		// make sure transactions on storages are properly finished, in
		// case previously it was unclean shutdown.
		err = m.verify(ctx)
		if err != nil {
			log.Warning(ctx, err)
			continue // -> recovery
		}
		log.Info(ctx, "verified ok")

		// provide service as long as partition table stays operational
		err = m.serve(ctx)
		if err != nil {
			log.Warning(ctx, err)
			continue // -> recovery
		}

		// TODO shutdown request ?
	}

	return ctx.Err()
}


// Cluster Recovery
// ----------------
//
// - starts from potentially no storage nodes known
// - accept connections from storage nodes
// - retrieve and recover latest previously saved partition table from storages
// - monitor whether partition table becomes operational wrt currently up nodeset
// - if yes - finish recovering upon receiving "start" command		TODO or autostart
// - start is also allowed if storages connected and say there is no partition
//   table saved to them (empty new cluster case).

// storRecovery is result of 1 storage node passing recovery phase.
type storRecovery struct {
	stor    *_MasteredPeer
	partTab *xneo.PartitionTable
	err     error

	// TODO + backup_tid, truncate_tid

	ack chan struct{} // main -> goStorCtlRecovery  "thanks; please continue"
}

// recovery drives cluster during recovery phase.
//
// when recovery finishes error indicates:
// - nil:  recovery was ok and a command came for cluster to start
// - !nil: recovery was cancelled
func (m *Master) recovery(ctx context.Context) (err error) {
	defer task.Running(&ctx, "recovery")(&err)

	m.setClusterState(ctx, proto.ClusterRecovering)
	ctx, rcancel := context.WithCancel(ctx)
	defer rcancel()

	recoveredq := make(chan storRecovery) // <- result of 1 stor recovery
	inprogress := 0                       // in-progress stor recoveries

	// requests to .ctlStart received when readyToStart=y
	// on success answered when full recovery completes
	startReqv := []chan error{}
	defer func() {
		errStart := ErrStartNonOperational
		if err == nil {
			errStart = nil
		}
		for _, ech := range startReqv {
			ech <- errStart
		}
	}()
//trace:event traceMasterStartReady(m *Master, ready bool)
	readyToStart := false // whether cluster currently can be operational or not
	updateReadyToStart := func() {
		var ready bool
		if m.node.State.PartTab.PTid == 0 {
			// new cluster - allow startup if we have some storages passed
			// recovery and there is no in-progress recovery running
			nup := 0
			for _, stor := range m.node.State.NodeTab.StorageList() {
				if stor.State > proto.DOWN {
					nup++
				}
			}
			ready = (nup > 0 && inprogress == 0)

		} else {
			ready = m.node.State.PartTab.OperationalWith(m.node.State.NodeTab)
		}

		if readyToStart != ready {
			state := "ready"
			if !ready {
				state = "not ready"
			}
			log.Infof(ctx, "cluster is %s to start", state)

			readyToStart = ready
			traceMasterStartReady(m, ready)

			// cluster became non-operational - cancel previously queued start requests
			if !ready {
				for _, ech := range startReqv {
					ech <- ErrStartNonOperational
				}
				startReqv = startReqv[:0]
			}
		}
	}

	// TODO down clients

	// goStorCtlRecovery spawns recovery task on a storage peer.
	goStorCtlRecovery := func(stor *_MasteredPeer) {
		inprogress++
		stor.wg.Go(func(peerCtx context.Context) error {
			ctx, cancel := xcontext.Merge/*Cancel*/(ctx, peerCtx)
			defer cancel()

			var pt *xneo.PartitionTable
			err := stor.run(ctx, func() error {
				var err error
				pt, err = storCtlRecovery(ctx, stor)
				return err
			})

			ack := make(chan struct{})
			recoveredq <- storRecovery{stor: stor, partTab: pt, err: err, ack: ack}
			<-ack

			// canceled recovery does not mean we should down the storage node
			if xxcontext.Canceled(err) {
				err = nil
			}
			return err
		})
	}

	// start recovery on all storages we are currently in touch with
	for _, peer := range m.peerTab {
		if peer.node.Type == proto.STORAGE {
			goStorCtlRecovery(peer)
		}
	}

	// during stop: react only to task completion and node leaving
	ctxDone   := ctx.Done()
	ctlStart  := m.ctlStart
	ctlStop   := m.ctlStop
	nodeComeq := m.nodeComeq
	err = nil
	stop := func(stopErr error) {
		if err != nil {
			return
		}
		err = stopErr
		rcancel()
		ctxDone   = nil
		ctlStart  = nil
		ctlStop   = nil
		nodeComeq = nil
	}

	for inprogress > 0 || !(
		/*start*/(readyToStart && len(startReqv) > 0) || /*stop*/(err != nil)) {

		select {
		case <-ctxDone:
			stop(ctx.Err())

		// request to start the cluster - if ok we queue it to reply ok on successful exit
		// if not ok - we just reply not ok
		case ech := <-ctlStart:
			if readyToStart {
				log.Infof(ctx, "start command - we are ready")
				// queue start request. Right now we believe we can
				// satisfy it, but during completion of spawned recovery
				// tasks, the cluster might become non-operational again.
				// If it will - queued start requests will be canceled.
				startReqv = append(startReqv, ech)
			} else {
				log.Infof(ctx, "start command - err - we are not ready")
				ech <- ErrStartNonOperational
			}

		case ech := <-ctlStop:
			close(ech) // ok; we are already recovering

		// peer (should be) disconnected
		case n := <-m.nodeLeaveq:
			m.disconnectPeer(ctx, n.peer)
			updateReadyToStart()

		// node comes in and asks to be identified
		case n := <-nodeComeq:
			peer, ok := m.identify(ctx, n,
				// TODO only accept:
				//	- S -> PENDING
				//	- M
			)
			if !ok {
				break
			}

			// S -> start recovery
			if peer.node.Type == proto.STORAGE {
				goStorCtlRecovery(peer)
			}

		// a storage node came through recovery - let's see whether
		// ptid ↑ and if so we should take partition table from there
		case r := <-recoveredq:
			close(r.ack) // for <-"node leave" to happen after <-recoveredq in case of err
			inprogress--

			if r.err != nil {
				// log error if it is not caused by verify cancel
				// TODO -> xxcontext.Canceled(err) but currently err is
				//         "...: use of closed network connection"
				if ctx.Err() == nil {
					log.Error(ctx, r.err)
				}
			} else {
				// we are interested in latest partTab
				// NOTE during recovery no one must be subscribed to
				// partTab so it is ok to simply change whole m.partTab
				// TODO -> updatePartTab()
				if r.partTab.PTid > m.node.State.PartTab.PTid {
					m.node.State.PartTab = r.partTab
				}
			}

			updateReadyToStart()
		}
	}

	if err != nil {
		return err
	}

	// recovery successful - we are starting

	// S PENDING -> RUNNING
	// XXX recheck logic is ok for when starting existing cluster
	for _, stor := range m.node.State.NodeTab.StorageList() {
		if stor.State == proto.PENDING {
			m.updateNodeState(ctx, stor, proto.RUNNING)
		}
	}

	// if we are starting for new cluster - create partition table
	if m.node.State.PartTab.PTid == 0 {
		storv := []*xneo.PeerNode{}
		for _, stor := range m.node.State.NodeTab.StorageList() {
			if stor.State > proto.DOWN {
				storv = append(storv, stor)
			}
		}
		m.node.State.PartTab = xneo.MakePartTab(
			1 /* np; XXX hardcoded */,
			1 /*R; XXX hardcoded*/,
			storv)
		m.node.State.PartTab.PTid = 1
		log.Infof(ctx, "creating new partition table: %s", m.node.State.PartTab)
	}

	return nil
}

// storCtlRecovery drives a storage node during cluster recovering state.
// it retrieves various ids and partition table from as stored on the storage
func storCtlRecovery(ctx context.Context, stor *_MasteredPeer) (_ *xneo.PartitionTable, err error) {
	slink := stor.node.Link()
	defer task.Runningf(&ctx, "rctl %s", stor.node.NID)(&err)

	// XXX cancel on ctx

	recovery := proto.AnswerRecovery{}
	err = slink.Ask1(&proto.Recovery{}, &recovery)
	if err != nil {
		return nil, err
	}

	resp := proto.AnswerPartitionTable{}
	err = slink.Ask1(&proto.AskPartitionTable{}, &resp)
	if err != nil {
		return nil, err
	}

	// reconstruct partition table from response
	// XXX  AnswerPartitionTable and SendPartitionTable is exactly the same
	// TODO -> better it be only 1 message
	mpt := &proto.SendPartitionTable{
		PTid:        resp.PTid,
		NumReplicas: resp.NumReplicas,
		RowList:     resp.RowList,
	}
	// NOTE NEO/py uses .NumReplicas as n(replica) - 1
	pt := xneo.PartTabFromMsg(mpt)
	return pt, nil
}


var errStopRequested   = errors.New("stop requested")
var errClusterDegraded = errors.New("cluster became non-operatonal")


// Cluster Verification (data recovery)
// ------------------------------------
//
// - starts with operational partition table
// - saves recovered partition table to all storages
// - asks all storages for partly finished transactions and decides cluster-wide
//   which such transactions need to be either finished or rolled back.
// - executes decided finishes and rollbacks on the storages.
// - retrieve last ids from storages along the way.
// - once we are done without losing too much storages in the process (so that
//   partition table is still operational) we are ready to enter servicing state.

// verify drives cluster during verification phase.
//
// when verify finishes error indicates:
// - nil:  verification completed ok; cluster is ready to enter running state
// - !nil: verification failed; cluster needs to be reset to recovery state
//
// prerequisite for start: .partTab is operational wrt .nodeTab
func (m *Master) verify(ctx context.Context) (err error) {
	defer task.Running(&ctx, "verify")(&err)

	m.setClusterState(ctx, proto.ClusterVerifying)
	ctx, vcancel := context.WithCancel(ctx)
	defer vcancel()

	verifiedq := make(chan storVerify) // <- result of stor verify task
	inprogress := 0                    // in-progress verify tasks

	// NOTE we don't reset m.lastOid / m.lastTid to 0 in the beginning of verification
	//      XXX (= py), rationale=?

	// goStorCtlVerify spawns verify task on a storage peer.
	goStorCtlVerify := func(stor *_MasteredPeer) {
		inprogress++
		stor.wg.Go(func(peerCtx context.Context) error {
			var lastOid zodb.Oid
			var lastTid zodb.Tid
			err := stor.run(ctx, func() error {
				var err error
				lastOid, lastTid, err = storCtlVerify(ctx, stor, m.node.State.PartTab)
				return err
			})

			ack := make(chan struct{})
			verifiedq <- storVerify{stor: stor, lastOid: lastOid, lastTid: lastTid, err: err, ack: ack}
			<-ack

			// canceled verify does not mean we should down the storage node
			if xxcontext.Canceled(err) {
				err = nil
			}
			return err
		})
	}

	// start verification on all storages we are currently in touch with
	for _, peer := range m.peerTab {
		if peer.node.Type == proto.STORAGE {
			goStorCtlVerify(peer)
		}
	}

	// during stop: react only to task completion and node leaving
	ctxDone   := ctx.Done()
	ctlStart  := m.ctlStart
	ctlStop   := m.ctlStop
	nodeComeq := m.nodeComeq
	err = nil
	stop := func(stopErr error) {
		if err != nil {
			return
		}
		err = stopErr
		vcancel()
		ctxDone   = nil
		ctlStart  = nil
		ctlStop   = nil
		nodeComeq = nil
	}

	for inprogress > 0 {
		select {
		case <-ctxDone:
			stop(ctx.Err())

		case ech := <-ctlStart:
			ech <- nil // we are already starting

		case ech := <-ctlStop:
			close(ech) // ok
			stop(errStopRequested)

		// peer (should be) disconnected
		case n := <-m.nodeLeaveq:
			m.disconnectPeer(ctx, n.peer)

			// if cluster became non-operational - we cancel verification
			if !m.node.State.PartTab.OperationalWith(m.node.State.NodeTab) {
				// XXX ok to instantly cancel? or better
				// graceful shutdown in-flight verifications?
				stop(errClusterDegraded)
			}

		// node comes in and asks to be identified
		case n := <-nodeComeq:
			peer, ok := m.identify(ctx, n,
				// TODO only accept:
				//	- S -> known ? RUNNING : PENDING
				//	- M
			)
			if !ok {
				break
			}

			// S -> start verification on it too
			if peer.node.Type == proto.STORAGE {
				goStorCtlVerify(peer)
			}


		// a storage node came through verification - adjust our last{Oid,Tid} if ok
		// on error check - whether cluster became non-operational and stop verification if so
		//
		// TODO actually implement logic to decide to finish/rollback transactions
		case v := <-verifiedq:
			close(v.ack) // for <-"node leave" to happen after <-verifiedq in case of err
			inprogress--

			if v.err != nil {
				// log error if it is not caused by verify cancel
				// TODO -> xxcontext.Canceled(err) but currently err is
				//         "...: use of closed network connection"
				if ctx.Err() == nil {
					log.Error(ctx, v.err)
				}

				// check partTab is still operational
				// if not -> cancel to go back to recovery
				if !m.node.State.PartTab.OperationalWith(m.node.State.NodeTab) {
					stop(errClusterDegraded)
				}
			} else {
				if v.lastOid > m.lastOid {
					m.lastOid = v.lastOid
				}
				if v.lastTid > m.lastTid {
					m.lastTid = v.lastTid
				}
			}
		}
	}

	return err
}

// storVerify is result of a storage node passing verification phase.
type storVerify struct {
	stor    *_MasteredPeer
	lastOid zodb.Oid
	lastTid zodb.Tid
	err	error

	ack chan struct{} // main -> goStorCtlVerify  "thanks; please continue"
}

// storCtlVerify drives a storage node during cluster verifying (= starting) state.
func storCtlVerify(ctx context.Context, stor *_MasteredPeer, pt *xneo.PartitionTable) (lastOid zodb.Oid, lastTid zodb.Tid, err error) {
	slink := stor.node.Link()
	defer task.Runningf(&ctx, "vctl %s", stor.node.NID)(&err)

	// XXX cancel on ctx

	lastOid = zodb.InvalidOid
	lastTid = zodb.InvalidTid

	// send just recovered parttab so storage saves it
	err = slink.Send1(pt.ToMsg())
	if err != nil {
		return
	}

	locked := proto.AnswerLockedTransactions{}
	err = slink.Ask1(&proto.LockedTransactions{}, &locked)
	if err != nil {
		return
	}

	if len(locked.TidDict) > 0 {
		err = fmt.Errorf("TODO: non-ø locked txns: %v", locked.TidDict)
		return
	}

	last := proto.AnswerLastIDs{}
	err = slink.Ask1(&proto.LastIDs{}, &last)
	if err != nil {
		return
	}

	return last.LastOid, last.LastTid, nil
}


// Cluster Running (operational service)
// -------------------------------------
//
// - starts with operational parttab and all present storage nodes consistently
//   either finished or rolled-back partly finished transactions.
// - monitor storages come & go and if parttab becomes non-operational leave to recovery.
// - provide service to clients while we are here.
//
// TODO also plan data movement on new storage nodes appearing

// serveDone is the error returned after serving peer is finished.
type serveDone struct {
	peer *_MasteredPeer
	err  error

	ack chan struct{} // main -> goServe  "thanks; please continue"
}

// serve drives cluster during running state.
//
// TODO document error meanings on return
//
// prerequisite for start: .partTab is operational wrt .nodeTab and verification passed
func (m *Master) serve(ctx context.Context) (err error) {
	defer task.Running(&ctx, "serve")(&err)

	m.setClusterState(ctx, proto.ClusterRunning)
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	servedq := make(chan serveDone) // <- result of a serve task
	inprogress := 0                 // in-progress serve tasks

	// goServe spawns serve task for a peer.
	goServe := func(peer *_MasteredPeer) {
		inprogress++
		peer.wg.Go(func(peerCtx context.Context) error {
			ctx, cancel := xcontext.Merge/*Cancel*/(ctx, peerCtx)
			defer cancel()

			switch peer.node.Type {
			case proto.STORAGE:
				err = storCtlServe(ctx, peer)

			case proto.CLIENT:
				err = m.serveClient(ctx, peer)

			// XXX ADMIN, MASTER
			default:
				err = fmt.Errorf("TODO: serve %s", peer.node.Type)
			}

			ack := make(chan struct{})
			servedq <- serveDone{peer: peer, err: err, ack: ack}
			<-ack

			// canceled serve does not necessarily mean we should down the peer
			if xxcontext.Canceled(err) {
				err = nil
			}
			return err
		})
	}

	// spawn peer serve driver (it should be only storages on entry here?)
	for _, peer := range m.peerTab {
		// XXX clients? other nodes?
		// XXX note PENDING - not adding to serve; ok?
		if peer.node.Type == proto.STORAGE && peer.node.State == proto.RUNNING {
			goServe(peer)
		}
	}

	// during stop: react only to task completion and node leaving
	ctxDone   := ctx.Done()
	ctlStart  := m.ctlStart
	ctlStop   := m.ctlStop
	nodeComeq := m.nodeComeq
	err = nil
	stop := func(stopErr error) {
		if err != nil {
			return
		}
		err = stopErr
		cancel()
		ctxDone   = nil
		ctlStart  = nil
		ctlStop   = nil
		nodeComeq = nil

		// TODO tell storages to stop serving
	}

	for inprogress > 0 {
		select {
		case <-ctxDone:
			stop(ctx.Err())

		case ech := <-ctlStart:
			ech <- nil // we are already started

		case ech := <-ctlStop:
			close(ech) // ok
			stop(fmt.Errorf("stop requested"))

		// peer (should be) disconnected
		case n := <-m.nodeLeaveq:
			m.disconnectPeer(ctx, n.peer)

			// if cluster became non-operational - cancel serve
			if !m.node.State.PartTab.OperationalWith(m.node.State.NodeTab) {
				stop(errClusterDegraded)
			}

		// new connection comes in -> start serving it
		case n := <-nodeComeq:
			peer, ok := m.identify(ctx, n, /* TODO accept everyone */)
			if !ok {
				break
			}

			goServe(peer)

		// serving a peer is done
		case d := <-servedq:
			close(d.ack) // for <-"node leave" to happen after <-servedq in case of err
			inprogress--

			if d.err != nil {
				// log error if it is not caused by verify cancel
				// TODO -> xxcontext.Canceled(err) but currently err is
				//         "...: use of closed network connection"
				if ctx.Err() == nil {
					log.Error(ctx, d.err)
				}
			}
		}
	}

	return err
}

// storCtlServe drives a storage node during cluster serve state.
func storCtlServe(ctx context.Context, stor *_MasteredPeer) (err error) {
	defer task.Runningf(&ctx, "sctl %s", stor.node.NID)(&err)
	slink := stor.node.Link()

	// XXX current neo/py does StartOperation / NotifyReady as separate
	//     sends, not exchange on the same conn. - py draftly fixed: see
	//     https://lab.nexedi.com/kirr/neo/commit/4eaaf186 for context.
	ready := proto.NotifyReady{}
	err = slink.Ask1(&proto.StartOperation{Backup: false}, &ready)
	//err = slink.Send1(&proto.StartOperation{Backup: false})
	//if err != nil {
	//	return err
	//}
	//req, err := slink.Recv1()
	//if err != nil {
	//	return err
	//}
	//msg := req.Msg
	//req.Close()
	//switch msg := msg.(type) {
	//case *proto.NotifyReady:
	//	// ok
	//case *proto.Error:
	//	return msg
	//default:
	//	return fmt.Errorf("unexpected message %T", msg)
	//}


	// now wait in a loop
	// TODO this should be also controlling transactions
	for {
		select {
		case <-time.After(1*time.Second):
			//println(".")

		case <-ctx.Done():
			// TODO also send StopOperation
			return ctx.Err()
		}
	}
}

// serveClient serves incoming client link.
func (m *Master) serveClient(ctx context.Context, cli *_MasteredPeer) (err error) {
	defer task.Runningf(&ctx, "serve %s", cli.node.NID)(&err)
	clink := cli.node.Link()

	// M <- C requests handler
	return xio.WithCloseOnRetCancel(ctx, clink, func() error {
		for {
			req, err := clink.Recv1()
			if err != nil {
				return err
			}

			resp := m.serveClient1(ctx, req.Msg)
			err = req.Reply(resp)
			req.Close()
			if err != nil {
				return err
			}
		}
	})
}

// serveClient1 prepares response for 1 request from client.
func (m *Master) serveClient1(ctx context.Context, req proto.Msg) (resp proto.Msg) {
	switch req := req.(type) {
	case *proto.LastTransaction:
		// XXX lock
		return &proto.AnswerLastTransaction{m.lastTid}

	default:
		return &proto.Error{proto.PROTOCOL_ERROR, fmt.Sprintf("unexpected message %T", req)}
	}
}

// ----------------------------------------

// disconnectPeer resets link to the peer and sets its state to DOWN in nodeTab.
// other peers are notified with δnodeTab about it.
func (m *Master) disconnectPeer(ctx context.Context, peer *_MasteredPeer) {
	log.Infof(ctx, "disconnecting %s", peer.node.NID)
	peer.cancel()
	peer.node.ResetLink(ctx)
	delete(m.peerTab, peer.node.NID)
	m.updateNodeState(ctx, peer.node, proto.DOWN)
}

// updateNodeTab = .nodeTab.Update  +  send δnodeTab to all subscribers.
// must be called from main.
func (m *Master) updateNodeTab(ctx context.Context, nodeInfo proto.NodeInfo) *xneo.PeerNode {
	log.Infof(ctx, "nodeTab[%s] <- %s", nodeInfo.NID, nodeInfo)
	node := m.node.State.NodeTab.Update(nodeInfo)
	m.notifyAll(ctx, &_ΔNodeTab{nodeInfo})
	return node
}

// updateNodeState updates nodeTab[peer.nid].state = state
func (m *Master) updateNodeState(ctx context.Context, peer *xneo.PeerNode, state proto.NodeState) {
	nodei := peer.NodeInfo
	nodei.State = state
	m.updateNodeTab(ctx, nodei)
}

// ----------------------------------------

// identify processes identification request of just connected node and either accepts or declines it.
//
// If node identification is accepted .nodeTab and .peerTab are updated and
// corresponding peer entry is returned. New task is spawned to reply with
// either reject or accept + notify.
func (m *Master) identify(ctx context.Context, n nodeCome) (peer *_MasteredPeer, ok bool) {
	nid := n.idReq.NID
	nodeType := n.idReq.NodeType

	err := func() *proto.Error {
		if n.idReq.ClusterName != m.node.ClusterName {
			return &proto.Error{proto.PROTOCOL_ERROR, "cluster name mismatch"}
		}

		if nid == 0 || nid == proto.NID(n.idReq.NodeType, 0) /* XXX <- stub for "temp" check */ {
			nid = m.allocNID(nodeType)
		}
		// TODO nid < 0 (temporary) -> reallocate if conflict ?
		// TODO check nid matches NodeType
		// TODO verify NodeType valid
		// XXX  handle IdTime ?


		node := m.node.State.NodeTab.Get(nid)
		if node != nil {
			// reject - nid is already occupied by someone else
			// TODO check also for down state - it could be the same node reconnecting
			return &proto.Error{proto.PROTOCOL_ERROR, fmt.Sprintf("nid %s already used by another node", nid)}
		}

		// accept only certain kind of nodes depending on .clusterState, e.g.
		// TODO caller should pass this logic to identify
		switch nodeType {
		case proto.CLIENT:
			if m.node.State.Code != proto.ClusterRunning {
				return &proto.Error{proto.NOT_READY, "cluster not operational"}
			}

		case proto.STORAGE:
			// ok

		// TODO +master, admin
		default:
			return &proto.Error{proto.PROTOCOL_ERROR, fmt.Sprintf("not accepting node type %s", nodeType)}
		}

		return nil
	}()

	link := n.req.Link()
	subj := fmt.Sprintf("identify: %s (%s)", link.RemoteAddr(), n.idReq.NID)
	if err != nil {
		log.Infof(ctx, "%s: rejecting: %s", subj, err)
		m.mainWG.Go(func(ctx context.Context) error {
			xio.WithCloseOnRetCancel(ctx, link, func() error {
				return n.req.Reply(err)
			})
			return nil // not to cancel main by a failing reject
		})
		return nil, false
	}

	log.Infof(ctx, "%s: accepting as %s", subj, nid)

	accept := &proto.AcceptIdentification{
		NodeType: proto.MASTER,
		MyNID:    m.node.MyInfo.NID,
		YourNID:  nid,
	}

	// update nodeTab
	var nodeState proto.NodeState
	switch nodeType {
	case proto.STORAGE:
		// FIXME py sets to RUNNING/PENDING depending on cluster state
		nodeState = proto.PENDING

	default:
		nodeState = proto.RUNNING
	}

	nodeInfo := proto.NodeInfo{
		Type:	nodeType,
		Addr:	n.idReq.Address,
		NID:	nid,
		State:	nodeState,
		IdTime:	proto.IdTime(m.monotime()),
	}

	node := m.updateNodeTab(ctx, nodeInfo)
	node.SetLink(link)


	// create peer with nodeTab/partTab snapshot to push to accepted node
	// and subscribe it for updates.
	peerCtx, peerCancel := context.WithCancel(m.runCtx)
	peer = &_MasteredPeer{
		node:      node,
		wg:        xsync.NewWorkGroup(peerCtx),
		cancel:    peerCancel,
		idReq:     n.req,
		acceptMsg: accept,
		state0:    m.node.State.Snapshot(),
		// TODO change limiting by buffer size -> to limiting by time
		//      (see notifyAll for details)
		notifyq:         make(chan _ΔClusterState, 1024),
		notifyqOverflow: make(chan struct{}),
		acceptSent:      make(chan struct{}),
	}
	m.peerTab[node.NID] = peer

	// spawn task to send accept and proxy δnodeTab/δpartTab to the peer
	peer.wg.Go(func(ctx context.Context) error {
		// go  main <- peer "peer (should be) disconnected"  when all peer's task finish
		m.mainWG.Go(func(ctx context.Context) error {
			// wait for all tasks related to peer to complete and then
			// notify main that peer node should go. Don't take ctx into
			// account - it is ~ runCtx and should be parent of context
			// under which per-peer tasks are spawned. This way if runCtx
			// is canceled -> any per-peer ctx should be canceled too and
			// wg.Wait should not block.
			err := peer.wg.Wait()

			select {
			case <-ctx.Done():
				return ctx.Err()
			case m.nodeLeaveq <- nodeLeave{peer, err}:
				return nil // ok
			}
		})

		// XXX compensate a bit for lack of ctx handling in Send/Recv
		return xio.WithCloseOnErrCancel(ctx, link, func() error {

			// send accept and indicate to run that initial acceptance is done
			err := peer.accept(ctx)
			if err != nil {
				return nil
			}
			close(peer.acceptSent)

			// proxy δnodeTab,δpartTab/δclusterState from main to the peer
			return peer.notify(ctx)
		})
	})

	return peer, true
}

// run runs f after initial phase of peer acceptance is over.
//
// XXX this is very similar if a separate Accept call would return peers
// already identified and answered with initial accept message sequence.
// However identification needs decisions from main task (to e.g. consult
// nodeTab to see if peer laddr is not overlapping with anyone's, and to assign
// nid). Because main is involved we cannot move it to completely separate task
// and give main only one Accept entry point to call.
func (p *_MasteredPeer) run(ctx context.Context, f func() error) error {
	select {
	case <-ctx.Done():
		return ctx.Err()
	// XXX in general we should also wait for if "accept0 failed". However
	// as that means accept0 task error, it would cancel ctx for all other
	// tasks run through p.wg . And run is called with contexts whose
	// cancel is derived from wg cancel - so we don't check for that.
	case <-p.acceptSent:
		return f()
	}
}

// accept sends accept reply and pushes inital state0 snapshot to peer.
func (p *_MasteredPeer) accept(ctx context.Context) (err error) {
	defer task.Runningf(&ctx, "accept %s", p.node.NID)(&err)

	link := p.node.Link()

	// send acceptance to just identified peer
	err = p.idReq.Reply(p.acceptMsg)
	if err != nil {
		return fmt.Errorf("send accept: %w", err)
	}

	// send initial state snapshot to accepted node

	// nodeTab
	err = link.Send1(&p.state0.NodeTab)
	if err != nil {
		return fmt.Errorf("send nodeTab: %w", err)
	}

	// partTab (not to S until cluster is RUNNING)
	if !(p.node.Type == proto.STORAGE && p.state0.Code != proto.ClusterRunning) {
		err = link.Send1(&p.state0.PartTab)
		if err != nil {
			return fmt.Errorf("send partTab: %w", err)
		}
	}

	// TODO send clusterState too? (NEO/py does not send it)
	return nil
}

// notifyAll notifies all peers about δstate event.
func (m *Master) notifyAll(ctx context.Context, event _ΔClusterState) {
	for nid, peer := range m.peerTab {
		// TODO change limiting by buffer size to limiting by time -
		// - i.e. detach peer if event queue grows more than 30s of time.
		select {
		case peer.notifyq <- event:
			continue // ok
		default:
		}

		log.Warningf(ctx, "peer %s is slow -> detaching it", nid)
		close(peer.notifyqOverflow)
		delete(m.peerTab, nid)
	}

}

// notify proxies δnodeTab/δpartTab/δClusterState update to the peer.
func (p *_MasteredPeer) notify(ctx context.Context) (err error) {
	defer task.Runningf(&ctx, "notify %s", p.node.NID)(&err)

	link := p.node.Link()
	stateCode := p.state0.Code

	for {
		var δstate _ΔClusterState

		select {
		case <-ctx.Done():
			return ctx.Err()

		case <-p.notifyqOverflow:
			return fmt.Errorf("detaching (peer is too slow to consume updates)")

		case δstate = <-p.notifyq:
		}

		var msg proto.Msg
		switch δstate := δstate.(type) {
		case *_ΔNodeTab:
			msg = &proto.NotifyNodeInformation{
				IdTime:   proto.IdTimeNone, // XXX ok?
				NodeList: []proto.NodeInfo{δstate.NodeInfo},
			}

		case *_ΔPartTab:
			// don't send δpartTab to S unless cluster is RUNNING
			if p.node.Type == proto.STORAGE && stateCode != proto.ClusterRunning {
				continue
			}
			msg = &proto.NotifyPartitionChanges{
				// XXX
				// PTid: ...,
				// NumReplicas: ...,
				// CellList: ...,
			}
			panic("TODO") // ^^^

		case *_ΔStateCode:
			stateCode = δstate.ClusterState
			msg = &proto.NotifyClusterState{
				State: stateCode,
			}

		default:
			panic("bug")
		}

		err := link.Send1(msg)
		if err != nil {
			return err
		}
	}
}

// allocNID allocates new node ID for a node of kind nodeType.
func (m *Master) allocNID(nodeType proto.NodeType) proto.NodeID {
	for num := int32(1); num < 1<<24; num++ {
		nid := proto.NID(nodeType, num)
		if m.node.State.NodeTab.Get(nid) == nil {
			return nid
		}
	}

	panic("all nid allocated ???")	// XXX more robust ?
}
