X kill mem (it uses unsafe inside)

Still crashes

wcfs/vendor/lab.nexedi.com/kirr/go123/mem/buffer.go

-// Copyright (C) 2017  Nexedi SA and Contributors.
-//                     Kirill Smelkov <kirr@nexedi.com>
-//
-// This program is free software: you can Use, Study, Modify and Redistribute
-// it under the terms of the GNU General Public License version 3, or (at your
-// option) any later version, as published by the Free Software Foundation.
-//
-// You can also Link and Combine this program with other software covered by
-// the terms of any of the Free Software licenses or any of the Open Source
-// Initiative approved licenses and Convey the resulting work. Corresponding
-// source of such a combination shall include the source code for all other
-// software used.
-//
-// This program is distributed WITHOUT ANY WARRANTY; without even the implied
-// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-//
-// See COPYING file for full licensing terms.
-// See https://www.nexedi.com/licensing for rationale and options.
-
-package mem
-// data buffers management
-
-import (
-	"sync"
-	"sync/atomic"
-
-	"lab.nexedi.com/kirr/go123/xmath"
-)
-
-// Buf is reference-counted memory buffer.
-//
-// To lower pressure on Go garbage-collector allocate buffers with BufAlloc and
-// free them with Buf.Release.
-//
-// Custom allocation functions affect only performance, not correctness -
-// everything should work if data buffer is allocated and/or free'ed
-// via regular Go/GC-way.
-type Buf struct {
-	Data []byte
-
-	// reference counter.
-	//
-	// NOTE to allow both Bufs created via BufAlloc and via std new, Buf is
-	// created with refcnt=0. The real number of references to Buf is thus .refcnt+1
-	refcnt int32
-}
-
-const order0 = 4 // buf sizes start from 2^4 (=16)
-
-var bufPoolv = [19]sync.Pool{} // buf size stop at 2^(4+19-1) (=4M)
-
-
-func init() {
-	for i := 0; i < len(bufPoolv); i++ {
-		i := i
-		bufPoolv[i].New = func() interface{} {
-			// NOTE *Buf, not just buf, to avoid allocation when
-			// making interface{} from it (interface{} wants to always point to heap)
-			return &Buf{Data: make([]byte, 1<<(order0+uint(i)))}
-		}
-	}
-}
-
-// BufAlloc allocates buffer of requested size from freelist.
-//
-// buffer memory is not initialized.
-func BufAlloc(size int) *Buf {
-	return BufAlloc64(int64(size))
-}
-
-// BufAlloc64 is same as BufAlloc but accepts int64 for size.
-func BufAlloc64(size int64) *Buf {
-	if size < 0 {
-		panic("invalid size")
-	}
-
-	// order = min i: 2^i >= size
-	order := xmath.CeilLog2(uint64(size))
-
-	order -= order0
-	if order < 0 {
-		order = 0
-	}
-
-	// if too big - allocate straightly from heap
-	if order >= len(bufPoolv) {
-		return &Buf{Data: make([]byte, size)}
-	}
-
-	buf := bufPoolv[order].Get().(*Buf)
-	buf.Data = buf.Data[:size] // leaving cap as is = 2^i
-	buf.refcnt = 0
-	return buf
-}
-
-// Release marks buf as no longer used by caller.
-//
-// It decrements buf reference-counter and if it reaches zero returns buf to
-// freelist.
-//
-// The caller must not use buf after call to Release.
-func (buf *Buf) Release() {
-	rc := atomic.AddInt32(&buf.refcnt, -1)
-	if rc < 0 - 1 {
-		panic("Buf.Release: refcnt < 0")
-	}
-	if rc > 0 - 1 {
-		return
-	}
-
-	// order = max i: 2^i <= cap
-	order := xmath.FloorLog2(uint64(cap(buf.Data)))
-
-	order -= order0
-	if order < 0 {
-		return // too small
-	}
-
-	if order >= len(bufPoolv) {
-		return // too big
-	}
-
-	bufPoolv[order].Put(buf)
-}
-
-// Incref increments buf's reference counter by 1.
-//
-// buf must already have reference-counter > 0 before Incref call.
-func (buf *Buf) Incref() {
-	rc := atomic.AddInt32(&buf.refcnt, +1)
-	if rc <= 1 - 1 {
-		panic("Buf.Incref: refcnt was < 1")
-	}
-}
-
-// XRelease releases buf it is != nil.
-func (buf *Buf) XRelease() {
-	if buf != nil {
-		buf.Release()
-	}
-}
-
-// XIncref increments buf's reference counter by 1 if buf != nil.
-func (buf *Buf) XIncref() {
-	if buf != nil {
-		buf.Incref()
-	}
-}
-
-
-// Len returns buf's len.
-//
-// it works even if buf=nil similarly to len() on nil []byte slice.
-func (buf *Buf) Len() int {
-	if buf != nil {
-		return len(buf.Data)
-	}
-	return 0
-}
-
-// Cap returns buf's cap.
-//
-// it works even if buf=nil similarly to len() on nil []byte slice.
-func (buf *Buf) Cap() int {
-	if buf != nil {
-		return cap(buf.Data)
-	}
-	return 0
-}
-
-// XData return's buf.Data or nil if buf == nil.
-func (buf *Buf) XData() []byte {
-	if buf != nil {
-		return buf.Data
-	}
-	return nil
-}

wcfs/vendor/lab.nexedi.com/kirr/go123/mem/buffer_test.s

-// empty .s so `go build` does not use -complete for go:linkname to work

wcfs/vendor/lab.nexedi.com/kirr/go123/mem/mem.go

-// Copyright (C) 2015-2017  Nexedi SA and Contributors.
-//                          Kirill Smelkov <kirr@nexedi.com>
-//
-// This program is free software: you can Use, Study, Modify and Redistribute
-// it under the terms of the GNU General Public License version 3, or (at your
-// option) any later version, as published by the Free Software Foundation.
-//
-// You can also Link and Combine this program with other software covered by
-// the terms of any of the Free Software licenses or any of the Open Source
-// Initiative approved licenses and Convey the resulting work. Corresponding
-// source of such a combination shall include the source code for all other
-// software used.
-//
-// This program is distributed WITHOUT ANY WARRANTY; without even the implied
-// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-//
-// See COPYING file for full licensing terms.
-// See https://www.nexedi.com/licensing for rationale and options.
-
-// Package mem provides reference-counted buffer and ways to work with memory
-// as either string or []byte without copying.
-package mem
-
-import (
-	"reflect"
-	"unsafe"
-)
-
-// Bytes converts string -> []byte without copying
-func Bytes(s string) []byte {
-	var b []byte
-	bp := (*reflect.SliceHeader)(unsafe.Pointer(&b))
-	bp.Data = (*reflect.StringHeader)(unsafe.Pointer(&s)).Data
-	bp.Cap = len(s)
-	bp.Len = len(s)
-	return b
-}
-
-// String converts []byte -> string without copying
-func String(b []byte) string {
-	var s string
-	sp := (*reflect.StringHeader)(unsafe.Pointer(&s))
-	sp.Data = (*reflect.SliceHeader)(unsafe.Pointer(&b)).Data
-	sp.Len = len(b)
-	return s
-}

wcfs/vendor/lab.nexedi.com/kirr/go123/xfmt/fmt.go

@@ -38,7 +38,6 @@ import (
 	"strconv"
 	"unicode/utf8"
 
-	"lab.nexedi.com/kirr/go123/mem"
 	"lab.nexedi.com/kirr/go123/xbytes"
 )
 
@@ -105,7 +104,7 @@ func (b *Buffer) Q(s string) *Buffer {
 
 // Qb appends []byte formatted by %q.
 func (b *Buffer) Qb(s []byte) *Buffer {
-	*b = strconv.AppendQuote(*b, mem.String(s))
+	*b = strconv.AppendQuote(*b, string(s))
 	return b
 }
 
@@ -175,7 +174,7 @@ func (b *Buffer) Xb(x []byte) *Buffer {
 
 // Xs appends string formatted by %x.
 func (b *Buffer) Xs(x string) *Buffer {
-	return b.Xb(mem.Bytes(x))
+	return b.Xb([]byte(x))
 }
 
 // TODO XX = %X

wcfs/vendor/lab.nexedi.com/kirr/go123/xfmt/python.go

@@ -25,13 +25,12 @@ import (
 	"strconv"
 	"unicode/utf8"
 
-	"lab.nexedi.com/kirr/go123/mem"
 	"lab.nexedi.com/kirr/go123/xbytes"
 )
 
 // AppendQuotePy appends to buf Python quoting of s.
 func AppendQuotePy(buf []byte, s string) []byte {
-	return AppendQuotePyBytes(buf, mem.Bytes(s))
+	return AppendQuotePyBytes(buf, []byte(s))
 }
 
 // AppendQuotePyBytes appends to buf Python quoting of b.

wcfs/vendor/lab.nexedi.com/kirr/neo/go/zodb/connection.go

@@ -25,7 +25,6 @@ import (
 	"fmt"
 	"sync"
 
-	"lab.nexedi.com/kirr/go123/mem"
 	"lab.nexedi.com/kirr/go123/xerr"
 	"lab.nexedi.com/kirr/neo/go/transaction"
 	"lab.nexedi.com/kirr/neo/go/zodb/internal/weak"
@@ -358,7 +357,7 @@ func (conn *Connection) Get(ctx context.Context, oid Oid) (_ IPersistent, err er
 }
 
 // load loads object specified by oid.
-func (conn *Connection) load(ctx context.Context, oid Oid) (_ *mem.Buf, serial Tid, _ error) {
+func (conn *Connection) load(ctx context.Context, oid Oid) (_ []byte, serial Tid, _ error) {
 	conn.checkTxnCtx(ctx, "load")
 	return conn.db.stor.Load(ctx, Xid{Oid: oid, At: conn.at})
 }

wcfs/vendor/lab.nexedi.com/kirr/neo/go/zodb/persistent.go

@@ -26,7 +26,6 @@ import (
 	"reflect"
 	"sync"
 
-	"lab.nexedi.com/kirr/go123/mem"
 	"lab.nexedi.com/kirr/go123/xerr"
 )
 
@@ -117,7 +116,7 @@ type Stateful interface {
 	// i.e. when the object has its in-RAM state.
 	//
 	// GetState should return a new buffer reference.
-	GetState() *mem.Buf
+	GetState() []byte
 
 	// SetState should set state of the in-RAM object from raw data.
 	//
@@ -129,7 +128,7 @@ type Stateful interface {
 	//
 	// The error returned does not need to have object/setstate prefix -
 	// persistent machinery is adding such prefix automatically.
-	SetState(state *mem.Buf) error
+	SetState(state []byte) error
 }
 
 // ---- RAM → DB: serialize ----
@@ -138,7 +137,7 @@ type Stateful interface {
 //
 // pSerialize is non-public method that is exposed and used only by ZODB internally.
 // pSerialize is called only on non-ghost objects.
-func (obj *Persistent) pSerialize() *mem.Buf {
+func (obj *Persistent) pSerialize() []byte {
 	obj.mu.Lock()
 	defer obj.mu.Unlock()
 	if obj.state == GHOST {
@@ -225,7 +224,7 @@ func (obj *Persistent) PActivate(ctx context.Context) (err error) {
 			panic(obj.badf("activate: !stateful instance"))
 		}
 
-		state.Release()
+		//state.Release()
 		if err == nil {
 			obj.state = UPTODATE
 		}
@@ -537,7 +536,7 @@ func persistentInit(xpobj reflect.Value, zc *zclass, jar *Connection, oid Oid, s
 type Broken struct {
 	Persistent
 	class string
-	state *mem.Buf
+	state []byte
 }
 
 // XXX register (Broken, brokenState) ?
@@ -546,18 +545,18 @@ var _ interface { Ghostable; Stateful} = (*brokenState)(nil)
 type brokenState Broken // hide state methods from public API
 
 func (b *brokenState) DropState() {
-	b.state.XRelease()
+	//b.state.XRelease()
 	b.state = nil
 }
 
-func (b *brokenState) GetState() *mem.Buf {
-	b.state.Incref()
+func (b *brokenState) GetState() []byte {
+	//b.state.Incref()
 	return b.state
 }
 
-func (b *brokenState) SetState(state *mem.Buf) error {
-	b.state.XRelease()
-	state.Incref()
+func (b *brokenState) SetState(state []byte) error {
+	//b.state.XRelease()
+	//state.Incref()
 	b.state = state
 	return nil
 }

wcfs/vendor/lab.nexedi.com/kirr/neo/go/zodb/storage.go

@@ -28,7 +28,6 @@ import (
 	"strings"
 	"sync"
 
-	"lab.nexedi.com/kirr/go123/mem"
 	"lab.nexedi.com/kirr/go123/xcontext"
 )
 
@@ -206,7 +205,7 @@ func (s *storage) Close() error {
 // loading goes through cache - this way prefetching can work
 
 // Load implements Loader.
-func (s *storage) Load(ctx context.Context, xid Xid) (*mem.Buf, Tid, error) {
+func (s *storage) Load(ctx context.Context, xid Xid) ([]byte, Tid, error) {
 	// XXX better -> xcontext.Merge(ctx, s.opCtx) but currently it costs 1+ goroutine
 	if ready(s.down) {
 		return nil, InvalidTid, s.zerr("load", xid, s.downErr)

wcfs/vendor/lab.nexedi.com/kirr/neo/go/zodb/storage/fs1/filestorage.go

@@ -75,7 +75,6 @@ import (
 
 	"lab.nexedi.com/kirr/neo/go/zodb"
 
-	"lab.nexedi.com/kirr/go123/mem"
 	"lab.nexedi.com/kirr/go123/xerr"
 
 	"github.com/fsnotify/fsnotify"
@@ -151,7 +150,7 @@ func (dh *DataHeader) Free() {
 }
 
 
-func (fs *FileStorage) Load(_ context.Context, xid zodb.Xid) (buf *mem.Buf, serial zodb.Tid, err error) {
+func (fs *FileStorage) Load(_ context.Context, xid zodb.Xid) (buf []byte, serial zodb.Tid, err error) {
 	// FIXME zodb.TidMax is only 7fff... tid from outside can be ffff...
 	// -> TODO reject tid out of range
 
@@ -164,7 +163,7 @@ func (fs *FileStorage) Load(_ context.Context, xid zodb.Xid) (buf *mem.Buf, seri
 //
 // FIXME kill Load_XXXWithNextSerialXXX after neo/py cache does not depend on next_serial
 // https://github.com/zopefoundation/ZODB/pull/323
-func (fs *FileStorage) Load_XXXWithNextSerialXXX(_ context.Context, xid zodb.Xid) (buf *mem.Buf, serial, nextSerial zodb.Tid, err error) {
+func (fs *FileStorage) Load_XXXWithNextSerialXXX(_ context.Context, xid zodb.Xid) (buf []byte, serial, nextSerial zodb.Tid, err error) {
 	buf, serial, nextSerial, err = fs.load(xid)
 	if err != nil {
 		err = fs.zerr("load", xid, err)
@@ -175,7 +174,7 @@ func (fs *FileStorage) Load_XXXWithNextSerialXXX(_ context.Context, xid zodb.Xid
 
 // FIXME kill nextSerial support after neo/py cache does not depend on next_serial
 // https://github.com/zopefoundation/ZODB/pull/323
-func (fs *FileStorage) load(xid zodb.Xid) (buf *mem.Buf, serial, nextSerial zodb.Tid, err error) {
+func (fs *FileStorage) load(xid zodb.Xid) (buf []byte, serial, nextSerial zodb.Tid, err error) {
 	// lookup in index position of oid data record within latest transaction which changed this oid
 	fs.mu.RLock()
 	if err := fs.downErr; err != nil {
@@ -222,7 +221,7 @@ func (fs *FileStorage) load(xid zodb.Xid) (buf *mem.Buf, serial, nextSerial zodb
 	if err != nil {
 		return nil, 0, 0, err
 	}
-	if buf.Data == nil {
+	if buf == nil {
 		// object was deleted
 		return nil, 0, 0, &zodb.NoDataError{Oid: xid.Oid, DeletedAt: serial}
 	}
@@ -248,7 +247,7 @@ type zIter struct {
 	dhLoading DataHeader
 
 	datai   zodb.DataInfo // ptr to this will be returned by .NextData
-	dataBuf *mem.Buf
+	dataBuf []byte
 }
 
 type zIterFlags int
@@ -300,7 +299,7 @@ func (zi *zIter) NextData(_ context.Context) (*zodb.DataInfo, error) {
 	// - need to use separate dh because of this.
 	zi.dhLoading = zi.iter.Datah
 	if zi.dataBuf != nil {
-		zi.dataBuf.Release()
+		//zi.dataBuf.Release()
 		zi.dataBuf = nil
 	}
 	zi.dataBuf, err = zi.dhLoading.LoadData(zi.iter.R)
@@ -308,7 +307,7 @@ func (zi *zIter) NextData(_ context.Context) (*zodb.DataInfo, error) {
 		return nil, err
 	}
 
-	zi.datai.Data = zi.dataBuf.Data
+	zi.datai.Data = zi.dataBuf
 	if zi.dhLoading.Tid != zi.datai.Tid {
 		zi.datai.DataTidHint = zi.dhLoading.Tid
 	} else {

wcfs/vendor/lab.nexedi.com/kirr/neo/go/zodb/storage/fs1/format.go

@@ -28,7 +28,6 @@ import (
 
 	"lab.nexedi.com/kirr/neo/go/zodb"
 
-	"lab.nexedi.com/kirr/go123/mem"
 	"lab.nexedi.com/kirr/go123/xbytes"
 )
 
@@ -667,23 +666,23 @@ func (dh *DataHeader) loadNext(r io.ReaderAt, txnh *TxnHeader) error {
 // On success dh state is changed to data header of original data transaction.
 //
 // NOTE "deleted" records are indicated via returning buf with .Data=nil without error.
-func (dh *DataHeader) LoadData(r io.ReaderAt) (*mem.Buf, error) {
+func (dh *DataHeader) LoadData(r io.ReaderAt) ([]byte, error) {
 	// scan via backpointers
 	for dh.DataLen == 0 {
 		err := dh.LoadBack(r)
 		if err != nil {
 			if err == io.EOF {
-				return &mem.Buf{Data: nil}, nil // deleted
+				return nil, nil // deleted
 			}
 			return nil, err
 		}
 	}
 
 	// now read actual data
-	buf := mem.BufAlloc64(dh.DataLen)
-	_, err := r.ReadAt(buf.Data, dh.Pos + DataHeaderSize)
+	buf := make([]byte, dh.DataLen)
+	_, err := r.ReadAt(buf, dh.Pos + DataHeaderSize)
 	if err != nil {
-		buf.Release()
+		//buf.Release()
 		return nil, dh.err(r, "read data", noEOF(err))
 	}
 

wcfs/vendor/lab.nexedi.com/kirr/neo/go/zodb/storage/fs1/index.go

@@ -38,7 +38,6 @@ import (
 	pickle "github.com/kisielk/og-rek"
 	"lab.nexedi.com/kirr/neo/go/zodb/internal/pickletools"
 
-	"lab.nexedi.com/kirr/go123/mem"
 	"lab.nexedi.com/kirr/go123/xbufio"
 	"lab.nexedi.com/kirr/go123/xerr"
 )
@@ -302,7 +301,7 @@ loop:
 		}
 
 		// load btree from fsBucket entries
-		kvBuf := mem.Bytes(kvStr)
+		kvBuf := []byte(kvStr)
 
 		n := len(kvBuf) / 8
 		oidBuf := kvBuf[:n*2]

wcfs/vendor/lab.nexedi.com/kirr/neo/go/zodb/zodb.go

@@ -170,8 +170,6 @@ package zodb
 import (
 	"context"
 	"fmt"
-
-	"lab.nexedi.com/kirr/go123/mem"
 )
 
 // ---- data model ----
@@ -442,7 +440,7 @@ type Loader interface {
 	// In ZODB/go Cache shows that it is possible to build efficient client
 	// cache without serial_next returned from Load. For this reason in ZODB/go
 	// Load specification comes without specifying serial_next return.
-	Load(ctx context.Context, xid Xid) (buf *mem.Buf, serial Tid, err error)
+	Load(ctx context.Context, xid Xid) (buf []byte, serial Tid, err error)
 }
 
 // Committer provides functionality to commit transactions.

wcfs/vendor/lab.nexedi.com/kirr/neo/go/zodb/zodbpy.go

@@ -22,8 +22,6 @@ package zodb
 
 import (
 	"context"
-
-	"lab.nexedi.com/kirr/go123/mem"
 )
 
 // PyStateful is the interface describing in-RAM object whose data state can be
@@ -54,8 +52,8 @@ type PyStateful interface {
 //
 // It is an error if decoded state has python class not as specified.
 // jar is used to resolve persistent references.
-func pySetState(obj PyStateful, objClass string, state *mem.Buf, jar *Connection) error {
-	pyclass, pystate, err := PyData(state.Data).decode(jar)
+func pySetState(obj PyStateful, objClass string, state []byte, jar *Connection) error {
+	pyclass, pystate, err := PyData(state).decode(jar)
 	if err != nil {
 		return err
 	}
@@ -72,11 +70,11 @@ func pySetState(obj PyStateful, objClass string, state *mem.Buf, jar *Connection
 }
 
 // pyGetState encodes obj as zodb/py serialized stream.
-func pyGetState(obj PyStateful, objClass string) *mem.Buf {
+func pyGetState(obj PyStateful, objClass string) []byte {
 	pyclass := zpyclass(objClass)
 	pystate := obj.PyGetState()
 	data := encodePyData(pyclass, pystate)
-	return &mem.Buf{Data: data} // XXX -> better bufalloc (and in encodePyData)
+	return data // XXX -> better bufalloc (and in encodePyData)
 }
 
 
@@ -91,9 +89,9 @@ func (conn *Connection) loadpy(ctx context.Context, oid Oid) (class string, pyst
 		return "", nil, 0, err
 	}
 
-	defer buf.Release()
+	//defer buf.Release()
 
-	pyclass, pystate, err := PyData(buf.Data).decode(conn)
+	pyclass, pystate, err := PyData(buf).decode(conn)
 	if err != nil {
 		err = &OpError{
 			URL:  conn.db.stor.URL(),