go/neo/neonet: Start (first draft)

Continue NEO/go with neonet - the layer to exchange messages in between
NEO nodes.

NEO/go shifts from thinking about NEO protocol logic as RPC to thinking
of it as more general network protocol and so settles to provide general
connection-oriented message exchange service. This way neonet provides
generic connection multiplexing on top of a single TCP node-node link.

Neonet compatibility with NEO/py depends on the following small NEO/py patch:

    dd3bb8b4

which adjusts message ID a bit so it behaves like stream_id in HTTP/2:

    - always even for server initiated streams
    - always odd  for client initiated streams

and is incremented by += 2, instead of += 1 to maintain above invariant.

See http://navytux.spb.ru/~kirr/neo.html#development-overview (starting from
"Then comes the link layer which provides service to exchange messages over
network...") for the rationale.

Unfortunately current NEO/py maintainer is very much against merging that patch.

This patch brings in the core of neonet. Next patches will add initial
handshaking, user-level Send/Recv + Ask/Expect and "lightweight mode".

Some neonet core history:

lab.nexedi.com/kirr/neo/commit/6b9ed46d	X neonet: Avoid integer overflow on max packet length check
lab.nexedi.com/kirr/neo/commit/8eac771c	X neo/connection: Fix race between link.shutdown() and conn.lightClose()
lab.nexedi.com/kirr/neo/commit/8021a1d5	X rxghandoff
lab.nexedi.com/kirr/neo/commit/68738036	X ... but negative impact on separate client / server processes, strange ...
lab.nexedi.com/kirr/neo/commit/b0dda9d2	X serveRecv: help Go scheduler to switch to receiving G sooner
lab.nexedi.com/kirr/neo/commit/4989918a	X remove defer from rx/tx hot paths
lab.nexedi.com/kirr/neo/commit/e055406a	X no select for acceptq - similarly for rxq path
lab.nexedi.com/kirr/neo/commit/c28ad4d0	X Conn.Recv: receive without select
lab.nexedi.com/kirr/neo/commit/496bd425	X add benchmark RTT over plain net.Conn with serveRecv-style RX handler
lab.nexedi.com/kirr/neo/commit/9fa79958	X draft how to mark RX down without reallocating .rxdown
lab.nexedi.com/kirr/neo/commit/4324c812	X restore all Conn functionality
lab.nexedi.com/kirr/neo/commit/a8e61d2f	X serveSend is not needed
lab.nexedi.com/kirr/neo/commit/9d047b36	X recvPkt via only 1 syscall
lab.nexedi.com/kirr/neo/commit/b555a507	X baseline net RTT benchmark
lab.nexedi.com/kirr/neo/commit/91be5cdd	X everyone is listening from start; CloseAccept to disable listening - works
lab.nexedi.com/kirr/neo/commit/c2a1b63a	X naming: Packet = raw data; Message = meaningful object
lab.nexedi.com/kirr/neo/commit/6fd0c9be	X connection: Adding context to errors from NodeLink and Conn operations
lab.nexedi.com/kirr/neo/commit/65b17bdc	X rework Conn acceptance to be explicit via NodeLink.Accept

go/neo/neonet/misc.go

+// Copyright (C) 2016-2018  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package neonet
+// syntax sugar for atomic load/store to raise signal/noise in logic
+
+import "sync/atomic"
+
+type atomic32 struct {
+	v int32	// struct member so `var a atomic32; if a == 0 ...` does not work
+}
+
+func (a *atomic32) Get() int32 {
+	return atomic.LoadInt32(&a.v)
+}
+
+func (a *atomic32) Set(v int32) {
+	atomic.StoreInt32(&a.v, v)
+}
+
+func (a *atomic32) Add(δ int32) int32 {
+	return atomic.AddInt32(&a.v, δ)
+}

go/neo/neonet/pkt.go

+// Copyright (C) 2016-2018  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package neonet
+// packets and packet buffers management
+
+import (
+	"fmt"
+	"reflect"
+	"sync"
+	"unsafe"
+
+	"lab.nexedi.com/kirr/go123/xbytes"
+
+	"lab.nexedi.com/kirr/neo/go/internal/packed"
+	"lab.nexedi.com/kirr/neo/go/neo/proto"
+)
+
+// pktBuf is a buffer with full raw packet (header + payload).
+//
+// Allocate pktBuf via pktAlloc() and free via pktBuf.Free().
+type pktBuf struct {
+	data []byte // whole packet data including all headers
+}
+
+// Header returns pointer to packet header.
+func (pkt *pktBuf) Header() *proto.PktHeader {
+	// NOTE no need to check len(.data) < PktHeader:
+	// .data is always allocated with cap >= PktHeaderLen.
+	return (*proto.PktHeader)(unsafe.Pointer(&pkt.data[0]))
+}
+
+// Payload returns []byte representing packet payload.
+func (pkt *pktBuf) Payload() []byte {
+	return pkt.data[proto.PktHeaderLen:]
+}
+
+// ---- pktBuf freelist ----
+
+// pktBufPool is sync.Pool<pktBuf>.
+var pktBufPool = sync.Pool{New: func() interface{} {
+	return &pktBuf{data: make([]byte, 0, 4096)}
+}}
+
+// pktAlloc allocates pktBuf with len=n.
+//
+// n must be >= sizeof(proto.PktHeader).
+func pktAlloc(n int) *pktBuf {
+	if n < proto.PktHeaderLen {
+		panic("pktAlloc: n < sizeof(PktHeader)")
+	}
+	pkt := pktBufPool.Get().(*pktBuf)
+	pkt.data = xbytes.Realloc(pkt.data, n)
+	return pkt
+}
+
+// Free marks pkt as no longer needed.
+func (pkt *pktBuf) Free() {
+	pktBufPool.Put(pkt)
+}
+
+
+// ---- pktBuf dump ----
+
+// String dumps a packet in human-readable form.
+func (pkt *pktBuf) String() string {
+	if len(pkt.data) < proto.PktHeaderLen {
+		return fmt.Sprintf("(! < PktHeaderLen) % x", pkt.data)
+	}
+
+	h := pkt.Header()
+	s := fmt.Sprintf(".%d", packed.Ntoh32(h.ConnId))
+
+	msgCode := packed.Ntoh16(h.MsgCode)
+	msgLen  := packed.Ntoh32(h.MsgLen)
+	data    := pkt.Payload()
+	msgType := proto.MsgType(msgCode)
+	if msgType == nil {
+		s += fmt.Sprintf(" ? (%d) #%d [%d]: % x", msgCode, msgLen, len(data), data)
+		return s
+	}
+
+	// XXX dup wrt Conn.Recv
+	msg := reflect.New(msgType).Interface().(proto.Msg)
+	n, err := msg.NEOMsgDecode(data)
+	if err != nil {
+		s += fmt.Sprintf(" (%s) %v; #%d [%d]: % x", msgType, err, msgLen, len(data), data)
+	}
+
+	s += fmt.Sprintf(" %s %v", msgType.Name(), msg) // XXX or %+v better?
+
+	if n < len(data) {
+		tail := data[n:]
+		s += fmt.Sprintf(" ;  [%d]tail: % x", len(tail), tail)
+	}
+
+	return s
+}
+
+// Dump dumps a packet in raw form.
+func (pkt *pktBuf) Dump() string {
+	if len(pkt.data) < proto.PktHeaderLen {
+		return fmt.Sprintf("(! < pktHeaderLen) % x", pkt.data)
+	}
+
+	h := pkt.Header()
+	data := pkt.Payload()
+	return fmt.Sprintf(".%d (%d) #%d [%d]: % x",
+		packed.Ntoh32(h.ConnId), packed.Ntoh16(h.MsgCode), packed.Ntoh32(h.MsgLen), len(data), data)
+}

go/neo/proto/proto.go

@@ -31,7 +31,8 @@
 // A message type can be looked up by message code with MsgType.
 //
 // The proto packages provides only message definitions and low-level
-// primitives for their marshalling.
+// primitives for their marshalling. Package lab.nexedi.com/kirr/neo/go/neo/neonet
+// provides actual service for message exchange over network.
 package proto
 
 // This file defines everything that relates to messages on the wire.