Skip to content

W
wendelin.core
Commit de1db2fe authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

X kill mem (it uses unsafe inside) 

Still crashes
parent 01942510
Hide whitespace changes
Inline Side-by-side
Showing with 36 additions and 272 deletions
wcfs/vendor/lab.nexedi.com/kirr/go123/mem/buffer.go deleted 100644 → 0
View file @ 01942510
// Copyright (C) 2017 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package mem
// data buffers management
import (
"sync"
"sync/atomic"
"lab.nexedi.com/kirr/go123/xmath"
)
// Buf is reference-counted memory buffer.
//
// To lower pressure on Go garbage-collector allocate buffers with BufAlloc and
// free them with Buf.Release.
//
// Custom allocation functions affect only performance, not correctness -
// everything should work if data buffer is allocated and/or free'ed
// via regular Go/GC-way.
type Buf struct {
Data []byte
// reference counter.
//
// NOTE to allow both Bufs created via BufAlloc and via std new, Buf is
// created with refcnt=0. The real number of references to Buf is thus .refcnt+1
refcnt int32
}
const order0 = 4 // buf sizes start from 2^4 (=16)
var bufPoolv = [19]sync.Pool{} // buf size stop at 2^(4+19-1) (=4M)
func init() {
for i := 0; i < len(bufPoolv); i++ {
i := i
bufPoolv[i].New = func() interface{} {
// NOTE *Buf, not just buf, to avoid allocation when
// making interface{} from it (interface{} wants to always point to heap)
return &Buf{Data: make([]byte, 1<<(order0+uint(i)))}
}
}
}
// BufAlloc allocates buffer of requested size from freelist.
//
// buffer memory is not initialized.
func BufAlloc(size int) *Buf {
return BufAlloc64(int64(size))
}
// BufAlloc64 is same as BufAlloc but accepts int64 for size.
func BufAlloc64(size int64) *Buf {
if size < 0 {
panic("invalid size")
}
// order = min i: 2^i >= size
order := xmath.CeilLog2(uint64(size))
order -= order0
if order < 0 {
order = 0
}
// if too big - allocate straightly from heap
if order >= len(bufPoolv) {
return &Buf{Data: make([]byte, size)}
}
buf := bufPoolv[order].Get().(*Buf)
buf.Data = buf.Data[:size] // leaving cap as is = 2^i
buf.refcnt = 0
return buf
}
// Release marks buf as no longer used by caller.
//
// It decrements buf reference-counter and if it reaches zero returns buf to
// freelist.
//
// The caller must not use buf after call to Release.
func (buf *Buf) Release() {
rc := atomic.AddInt32(&buf.refcnt, -1)
if rc < 0 - 1 {
panic("Buf.Release: refcnt < 0")
}
if rc > 0 - 1 {
return
}
// order = max i: 2^i <= cap
order := xmath.FloorLog2(uint64(cap(buf.Data)))
order -= order0
if order < 0 {
return // too small
}
if order >= len(bufPoolv) {
return // too big
}
bufPoolv[order].Put(buf)
}
// Incref increments buf's reference counter by 1.
//
// buf must already have reference-counter > 0 before Incref call.
func (buf *Buf) Incref() {
rc := atomic.AddInt32(&buf.refcnt, +1)
if rc <= 1 - 1 {
panic("Buf.Incref: refcnt was < 1")
}
}
// XRelease releases buf it is != nil.
func (buf *Buf) XRelease() {
if buf != nil {
buf.Release()
}
}
// XIncref increments buf's reference counter by 1 if buf != nil.
func (buf *Buf) XIncref() {
if buf != nil {
buf.Incref()
}
}
// Len returns buf's len.
//
// it works even if buf=nil similarly to len() on nil []byte slice.
func (buf *Buf) Len() int {
if buf != nil {
return len(buf.Data)
}
return 0
}
// Cap returns buf's cap.
//
// it works even if buf=nil similarly to len() on nil []byte slice.
func (buf *Buf) Cap() int {
if buf != nil {
return cap(buf.Data)
}
return 0
}
// XData return's buf.Data or nil if buf == nil.
func (buf *Buf) XData() []byte {
if buf != nil {
return buf.Data
}
return nil
}
wcfs/vendor/lab.nexedi.com/kirr/go123/mem/buffer_test.s deleted 100644 → 0
View file @ 01942510
// empty .s so `go build` does not use -complete for go:linkname to work
wcfs/vendor/lab.nexedi.com/kirr/go123/mem/mem.go deleted 100644 → 0
View file @ 01942510
// Copyright (C) 2015-2017 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
// Package mem provides reference-counted buffer and ways to work with memory
// as either string or []byte without copying.
package mem
import (
"reflect"
"unsafe"
)
// Bytes converts string -> []byte without copying
func Bytes(s string) []byte {
var b []byte
bp := (*reflect.SliceHeader)(unsafe.Pointer(&b))
bp.Data = (*reflect.StringHeader)(unsafe.Pointer(&s)).Data
bp.Cap = len(s)
bp.Len = len(s)
return b
}
// String converts []byte -> string without copying
func String(b []byte) string {
var s string
sp := (*reflect.StringHeader)(unsafe.Pointer(&s))
sp.Data = (*reflect.SliceHeader)(unsafe.Pointer(&b)).Data
sp.Len = len(b)
return s
}
wcfs/vendor/lab.nexedi.com/kirr/go123/xfmt/fmt.go
View file @ de1db2fe
......@@ -38,7 +38,6 @@ import (
"strconv"
"unicode/utf8"
"lab.nexedi.com/kirr/go123/mem"
"lab.nexedi.com/kirr/go123/xbytes"
)
......@@ -105,7 +104,7 @@ func (b *Buffer) Q(s string) *Buffer {
// Qb appends []byte formatted by %q.
func (b *Buffer) Qb(s []byte) *Buffer {
*b = strconv.AppendQuote(*b, mem.String(s))
*b = strconv.AppendQuote(*b, string(s))
return b
}
......@@ -175,7 +174,7 @@ func (b *Buffer) Xb(x []byte) *Buffer {
// Xs appends string formatted by %x.
func (b *Buffer) Xs(x string) *Buffer {
return b.Xb(mem.Bytes(x))
return b.Xb([]byte(x))
}
// TODO XX = %X
......
wcfs/vendor/lab.nexedi.com/kirr/go123/xfmt/python.go
View file @ de1db2fe
......@@ -25,13 +25,12 @@ import (
"strconv"
"unicode/utf8"
"lab.nexedi.com/kirr/go123/mem"
"lab.nexedi.com/kirr/go123/xbytes"
)
// AppendQuotePy appends to buf Python quoting of s.
func AppendQuotePy(buf []byte, s string) []byte {
return AppendQuotePyBytes(buf, mem.Bytes(s))
return AppendQuotePyBytes(buf, []byte(s))
}
// AppendQuotePyBytes appends to buf Python quoting of b.
......
wcfs/vendor/lab.nexedi.com/kirr/neo/go/zodb/connection.go
View file @ de1db2fe
......@@ -25,7 +25,6 @@ import (
"fmt"
"sync"
"lab.nexedi.com/kirr/go123/mem"
"lab.nexedi.com/kirr/go123/xerr"
"lab.nexedi.com/kirr/neo/go/transaction"
"lab.nexedi.com/kirr/neo/go/zodb/internal/weak"
......@@ -358,7 +357,7 @@ func (conn *Connection) Get(ctx context.Context, oid Oid) (_ IPersistent, err er
}
// load loads object specified by oid.
func (conn *Connection) load(ctx context.Context, oid Oid) (_ *mem.Buf, serial Tid, _ error) {
func (conn *Connection) load(ctx context.Context, oid Oid) (_ []byte, serial Tid, _ error) {
conn.checkTxnCtx(ctx, "load")
return conn.db.stor.Load(ctx, Xid{Oid: oid, At: conn.at})
}
......
wcfs/vendor/lab.nexedi.com/kirr/neo/go/zodb/persistent.go
View file @ de1db2fe
......@@ -26,7 +26,6 @@ import (
"reflect"
"sync"
"lab.nexedi.com/kirr/go123/mem"
"lab.nexedi.com/kirr/go123/xerr"
)
......@@ -117,7 +116,7 @@ type Stateful interface {
// i.e. when the object has its in-RAM state.
//
// GetState should return a new buffer reference.
GetState() *mem.Buf
GetState() []byte
// SetState should set state of the in-RAM object from raw data.
//
......@@ -129,7 +128,7 @@ type Stateful interface {
//
// The error returned does not need to have object/setstate prefix -
// persistent machinery is adding such prefix automatically.
SetState(state *mem.Buf) error
SetState(state []byte) error
}
// ---- RAM → DB: serialize ----
......@@ -138,7 +137,7 @@ type Stateful interface {
//
// pSerialize is non-public method that is exposed and used only by ZODB internally.
// pSerialize is called only on non-ghost objects.
func (obj *Persistent) pSerialize() *mem.Buf {
func (obj *Persistent) pSerialize() []byte {
obj.mu.Lock()
defer obj.mu.Unlock()
if obj.state == GHOST {
......@@ -225,7 +224,7 @@ func (obj *Persistent) PActivate(ctx context.Context) (err error) {
panic(obj.badf("activate: !stateful instance"))
}
state.Release()
//state.Release()
if err == nil {
obj.state = UPTODATE
}
......@@ -537,7 +536,7 @@ func persistentInit(xpobj reflect.Value, zc *zclass, jar *Connection, oid Oid, s
type Broken struct {
Persistent
class string
state *mem.Buf
state []byte
}
// XXX register (Broken, brokenState) ?
......@@ -546,18 +545,18 @@ var _ interface { Ghostable; Stateful} = (*brokenState)(nil)
type brokenState Broken // hide state methods from public API
func (b *brokenState) DropState() {
b.state.XRelease()
//b.state.XRelease()
b.state = nil
}
func (b *brokenState) GetState() *mem.Buf {
b.state.Incref()
func (b *brokenState) GetState() []byte {
//b.state.Incref()
return b.state
}
func (b *brokenState) SetState(state *mem.Buf) error {
b.state.XRelease()
state.Incref()
func (b *brokenState) SetState(state []byte) error {
//b.state.XRelease()
//state.Incref()
b.state = state
return nil
}
......
wcfs/vendor/lab.nexedi.com/kirr/neo/go/zodb/storage.go
View file @ de1db2fe
......@@ -28,7 +28,6 @@ import (
"strings"
"sync"
"lab.nexedi.com/kirr/go123/mem"
"lab.nexedi.com/kirr/go123/xcontext"
)
......@@ -206,7 +205,7 @@ func (s *storage) Close() error {
// loading goes through cache - this way prefetching can work
// Load implements Loader.
func (s *storage) Load(ctx context.Context, xid Xid) (*mem.Buf, Tid, error) {
func (s *storage) Load(ctx context.Context, xid Xid) ([]byte, Tid, error) {
// XXX better -> xcontext.Merge(ctx, s.opCtx) but currently it costs 1+ goroutine
if ready(s.down) {
return nil, InvalidTid, s.zerr("load", xid, s.downErr)
......
wcfs/vendor/lab.nexedi.com/kirr/neo/go/zodb/storage/fs1/filestorage.go
View file @ de1db2fe
......@@ -75,7 +75,6 @@ import (
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/kirr/go123/mem"
"lab.nexedi.com/kirr/go123/xerr"
"github.com/fsnotify/fsnotify"
......@@ -151,7 +150,7 @@ func (dh *DataHeader) Free() {
}
func (fs *FileStorage) Load(_ context.Context, xid zodb.Xid) (buf *mem.Buf, serial zodb.Tid, err error) {
func (fs *FileStorage) Load(_ context.Context, xid zodb.Xid) (buf []byte, serial zodb.Tid, err error) {
// FIXME zodb.TidMax is only 7fff... tid from outside can be ffff...
// -> TODO reject tid out of range
......@@ -164,7 +163,7 @@ func (fs *FileStorage) Load(_ context.Context, xid zodb.Xid) (buf *mem.Buf, seri
//
// FIXME kill Load_XXXWithNextSerialXXX after neo/py cache does not depend on next_serial
// https://github.com/zopefoundation/ZODB/pull/323
func (fs *FileStorage) Load_XXXWithNextSerialXXX(_ context.Context, xid zodb.Xid) (buf *mem.Buf, serial, nextSerial zodb.Tid, err error) {
func (fs *FileStorage) Load_XXXWithNextSerialXXX(_ context.Context, xid zodb.Xid) (buf []byte, serial, nextSerial zodb.Tid, err error) {
buf, serial, nextSerial, err = fs.load(xid)
if err != nil {
err = fs.zerr("load", xid, err)
......@@ -175,7 +174,7 @@ func (fs *FileStorage) Load_XXXWithNextSerialXXX(_ context.Context, xid zodb.Xid
// FIXME kill nextSerial support after neo/py cache does not depend on next_serial
// https://github.com/zopefoundation/ZODB/pull/323
func (fs *FileStorage) load(xid zodb.Xid) (buf *mem.Buf, serial, nextSerial zodb.Tid, err error) {
func (fs *FileStorage) load(xid zodb.Xid) (buf []byte, serial, nextSerial zodb.Tid, err error) {
// lookup in index position of oid data record within latest transaction which changed this oid
fs.mu.RLock()
if err := fs.downErr; err != nil {
......@@ -222,7 +221,7 @@ func (fs *FileStorage) load(xid zodb.Xid) (buf *mem.Buf, serial, nextSerial zodb
if err != nil {
return nil, 0, 0, err
}
if buf.Data == nil {
if buf == nil {
// object was deleted
return nil, 0, 0, &zodb.NoDataError{Oid: xid.Oid, DeletedAt: serial}
}
......@@ -248,7 +247,7 @@ type zIter struct {
dhLoading DataHeader
datai zodb.DataInfo // ptr to this will be returned by .NextData
dataBuf *mem.Buf
dataBuf []byte
}
type zIterFlags int
......@@ -300,7 +299,7 @@ func (zi *zIter) NextData(_ context.Context) (*zodb.DataInfo, error) {
// - need to use separate dh because of this.
zi.dhLoading = zi.iter.Datah
if zi.dataBuf != nil {
zi.dataBuf.Release()
//zi.dataBuf.Release()
zi.dataBuf = nil
}
zi.dataBuf, err = zi.dhLoading.LoadData(zi.iter.R)
......@@ -308,7 +307,7 @@ func (zi *zIter) NextData(_ context.Context) (*zodb.DataInfo, error) {
return nil, err
}
zi.datai.Data = zi.dataBuf.Data
zi.datai.Data = zi.dataBuf
if zi.dhLoading.Tid != zi.datai.Tid {
zi.datai.DataTidHint = zi.dhLoading.Tid
} else {
......
wcfs/vendor/lab.nexedi.com/kirr/neo/go/zodb/storage/fs1/format.go
View file @ de1db2fe
......@@ -28,7 +28,6 @@ import (
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/kirr/go123/mem"
"lab.nexedi.com/kirr/go123/xbytes"
)
......@@ -667,23 +666,23 @@ func (dh *DataHeader) loadNext(r io.ReaderAt, txnh *TxnHeader) error {
// On success dh state is changed to data header of original data transaction.
//
// NOTE "deleted" records are indicated via returning buf with .Data=nil without error.
func (dh *DataHeader) LoadData(r io.ReaderAt) (*mem.Buf, error) {
func (dh *DataHeader) LoadData(r io.ReaderAt) ([]byte, error) {
// scan via backpointers
for dh.DataLen == 0 {
err := dh.LoadBack(r)
if err != nil {
if err == io.EOF {
return &mem.Buf{Data: nil}, nil // deleted
return nil, nil // deleted
}
return nil, err
}
}
// now read actual data
buf := mem.BufAlloc64(dh.DataLen)
_, err := r.ReadAt(buf.Data, dh.Pos + DataHeaderSize)
buf := make([]byte, dh.DataLen)
_, err := r.ReadAt(buf, dh.Pos + DataHeaderSize)
if err != nil {
buf.Release()
//buf.Release()
return nil, dh.err(r, "read data", noEOF(err))
}
......
wcfs/vendor/lab.nexedi.com/kirr/neo/go/zodb/storage/fs1/index.go
View file @ de1db2fe
......@@ -38,7 +38,6 @@ import (
pickle "github.com/kisielk/og-rek"
"lab.nexedi.com/kirr/neo/go/zodb/internal/pickletools"
"lab.nexedi.com/kirr/go123/mem"
"lab.nexedi.com/kirr/go123/xbufio"
"lab.nexedi.com/kirr/go123/xerr"
)
......@@ -302,7 +301,7 @@ loop:
}
// load btree from fsBucket entries
kvBuf := mem.Bytes(kvStr)
kvBuf := []byte(kvStr)
n := len(kvBuf) / 8
oidBuf := kvBuf[:n*2]
......
wcfs/vendor/lab.nexedi.com/kirr/neo/go/zodb/zodb.go
View file @ de1db2fe
......@@ -170,8 +170,6 @@ package zodb
import (
"context"
"fmt"
"lab.nexedi.com/kirr/go123/mem"
)
// ---- data model ----
......@@ -442,7 +440,7 @@ type Loader interface {
// In ZODB/go Cache shows that it is possible to build efficient client
// cache without serial_next returned from Load. For this reason in ZODB/go
// Load specification comes without specifying serial_next return.
Load(ctx context.Context, xid Xid) (buf *mem.Buf, serial Tid, err error)
Load(ctx context.Context, xid Xid) (buf []byte, serial Tid, err error)
}
// Committer provides functionality to commit transactions.
......
wcfs/vendor/lab.nexedi.com/kirr/neo/go/zodb/zodbpy.go
View file @ de1db2fe
......@@ -22,8 +22,6 @@ package zodb
import (
"context"
"lab.nexedi.com/kirr/go123/mem"
)
// PyStateful is the interface describing in-RAM object whose data state can be
......@@ -54,8 +52,8 @@ type PyStateful interface {
//
// It is an error if decoded state has python class not as specified.
// jar is used to resolve persistent references.
func pySetState(obj PyStateful, objClass string, state *mem.Buf, jar *Connection) error {
pyclass, pystate, err := PyData(state.Data).decode(jar)
func pySetState(obj PyStateful, objClass string, state []byte, jar *Connection) error {
pyclass, pystate, err := PyData(state).decode(jar)
if err != nil {
return err
}
......@@ -72,11 +70,11 @@ func pySetState(obj PyStateful, objClass string, state *mem.Buf, jar *Connection
}
// pyGetState encodes obj as zodb/py serialized stream.
func pyGetState(obj PyStateful, objClass string) *mem.Buf {
func pyGetState(obj PyStateful, objClass string) []byte {
pyclass := zpyclass(objClass)
pystate := obj.PyGetState()
data := encodePyData(pyclass, pystate)
return &mem.Buf{Data: data} // XXX -> better bufalloc (and in encodePyData)
return data // XXX -> better bufalloc (and in encodePyData)
}
......@@ -91,9 +89,9 @@ func (conn *Connection) loadpy(ctx context.Context, oid Oid) (class string, pyst
return "", nil, 0, err
}
defer buf.Release()
//defer buf.Release()
pyclass, pystate, err := PyData(buf.Data).decode(conn)
pyclass, pystate, err := PyData(buf).decode(conn)
if err != nil {
err = &OpError{
URL: conn.db.stor.URL(),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7