Skip to content

J
jacobsa-fuse
Commit 641629d1 authored by Aaron Jacobs's avatar Aaron Jacobs
Browse files
Options

buffer: refactor to a form that won't require memclr. 

Necessary because our evil memclr should go away.

Benchmarks:

    benchmark                                          old ns/op     new ns/op     delta
    BenchmarkOutMessageReset/Single_buffer-12          4.94          4.95          +0.20%
    BenchmarkOutMessageReset/Many_buffers-12           10.4          12.0          +15.38%
    BenchmarkOutMessageGrowShrink/Single_buffer-12     42861         47666         +11.21%
    BenchmarkOutMessageGrowShrink/Many_buffers-12      101896        100626        -1.25%

    benchmark                                          old MB/s     new MB/s     speedup
    BenchmarkOutMessageReset/Single_buffer-12          4853.85      4851.64      1.00x
    BenchmarkOutMessageReset/Many_buffers-12           2311.82      2006.72      0.87x
    BenchmarkOutMessageGrowShrink/Single_buffer-12     24464.39     21998.19     0.90x
    BenchmarkOutMessageGrowShrink/Many_buffers-12      10290.60     10420.48     1.01x
parents 10bc0e76 95e4d8b1
Hide whitespace changes
Inline Side-by-side
Showing with 99 additions and 88 deletions
conversions.go
View file @ 641629d1
......@@ -279,7 +279,7 @@ func convertInMessage(
o = to
readSize := int(in.Size)
p := outMsg.GrowNoZero(uintptr(readSize))
p := outMsg.GrowNoZero(readSize)
if p == nil {
err = fmt.Errorf("Can't grow for %d-byte read", readSize)
return
......@@ -305,7 +305,7 @@ func convertInMessage(
o = to
readSize := int(in.Size)
p := outMsg.GrowNoZero(uintptr(readSize))
p := outMsg.GrowNoZero(readSize)
if p == nil {
err = fmt.Errorf("Can't grow for %d-byte read", readSize)
return
......@@ -469,7 +469,7 @@ func (c *Connection) kernelResponse(
// the header, because on OS X the kernel otherwise returns EINVAL when we
// attempt to write an error response with a length that extends beyond the
// header.
m.ShrinkTo(buffer.OutMessageInitialSize)
m.ShrinkTo(buffer.OutMessageHeaderSize)
}
// Otherwise, fill in the rest of the response.
......@@ -489,45 +489,45 @@ func (c *Connection) kernelResponseForOp(
// Create the appropriate output message
switch o := op.(type) {
case *fuseops.LookUpInodeOp:
size := fusekernel.EntryOutSize(c.protocol)
size := int(fusekernel.EntryOutSize(c.protocol))
out := (*fusekernel.EntryOut)(m.Grow(size))
convertChildInodeEntry(&o.Entry, out)
case *fuseops.GetInodeAttributesOp:
size := fusekernel.AttrOutSize(c.protocol)
size := int(fusekernel.AttrOutSize(c.protocol))
out := (*fusekernel.AttrOut)(m.Grow(size))
out.AttrValid, out.AttrValidNsec = convertExpirationTime(
o.AttributesExpiration)
convertAttributes(o.Inode, &o.Attributes, &out.Attr)
case *fuseops.SetInodeAttributesOp:
size := fusekernel.AttrOutSize(c.protocol)
size := int(fusekernel.AttrOutSize(c.protocol))
out := (*fusekernel.AttrOut)(m.Grow(size))
out.AttrValid, out.AttrValidNsec = convertExpirationTime(
o.AttributesExpiration)
convertAttributes(o.Inode, &o.Attributes, &out.Attr)
case *fuseops.MkDirOp:
size := fusekernel.EntryOutSize(c.protocol)
size := int(fusekernel.EntryOutSize(c.protocol))
out := (*fusekernel.EntryOut)(m.Grow(size))
convertChildInodeEntry(&o.Entry, out)
case *fuseops.MkNodeOp:
size := fusekernel.EntryOutSize(c.protocol)
size := int(fusekernel.EntryOutSize(c.protocol))
out := (*fusekernel.EntryOut)(m.Grow(size))
convertChildInodeEntry(&o.Entry, out)
case *fuseops.CreateFileOp:
eSize := fusekernel.EntryOutSize(c.protocol)
eSize := int(fusekernel.EntryOutSize(c.protocol))
e := (*fusekernel.EntryOut)(m.Grow(eSize))
convertChildInodeEntry(&o.Entry, e)
oo := (*fusekernel.OpenOut)(m.Grow(unsafe.Sizeof(fusekernel.OpenOut{})))
oo := (*fusekernel.OpenOut)(m.Grow(int(unsafe.Sizeof(fusekernel.OpenOut{}))))
oo.Fh = uint64(o.Handle)
case *fuseops.CreateSymlinkOp:
size := fusekernel.EntryOutSize(c.protocol)
size := int(fusekernel.EntryOutSize(c.protocol))
out := (*fusekernel.EntryOut)(m.Grow(size))
convertChildInodeEntry(&o.Entry, out)
......@@ -541,20 +541,20 @@ func (c *Connection) kernelResponseForOp(
// Empty response
case *fuseops.OpenDirOp:
out := (*fusekernel.OpenOut)(m.Grow(unsafe.Sizeof(fusekernel.OpenOut{})))
out := (*fusekernel.OpenOut)(m.Grow(int(unsafe.Sizeof(fusekernel.OpenOut{}))))
out.Fh = uint64(o.Handle)
case *fuseops.ReadDirOp:
// convertInMessage already set up the destination buffer to be at the end
// of the out message. We need only shrink to the right size based on how
// much the user read.
m.ShrinkTo(buffer.OutMessageInitialSize + uintptr(o.BytesRead))
m.ShrinkTo(buffer.OutMessageHeaderSize + o.BytesRead)
case *fuseops.ReleaseDirHandleOp:
// Empty response
case *fuseops.OpenFileOp:
out := (*fusekernel.OpenOut)(m.Grow(unsafe.Sizeof(fusekernel.OpenOut{})))
out := (*fusekernel.OpenOut)(m.Grow(int(unsafe.Sizeof(fusekernel.OpenOut{}))))
out.Fh = uint64(o.Handle)
if o.KeepPageCache {
......@@ -565,10 +565,10 @@ func (c *Connection) kernelResponseForOp(
// convertInMessage already set up the destination buffer to be at the end
// of the out message. We need only shrink to the right size based on how
// much the user read.
m.ShrinkTo(buffer.OutMessageInitialSize + uintptr(o.BytesRead))
m.ShrinkTo(buffer.OutMessageHeaderSize + o.BytesRead)
case *fuseops.WriteFileOp:
out := (*fusekernel.WriteOut)(m.Grow(unsafe.Sizeof(fusekernel.WriteOut{})))
out := (*fusekernel.WriteOut)(m.Grow(int(unsafe.Sizeof(fusekernel.WriteOut{}))))
out.Size = uint32(len(o.Data))
case *fuseops.SyncFileOp:
......@@ -584,7 +584,7 @@ func (c *Connection) kernelResponseForOp(
m.AppendString(o.Target)
case *fuseops.StatFSOp:
out := (*fusekernel.StatfsOut)(m.Grow(unsafe.Sizeof(fusekernel.StatfsOut{})))
out := (*fusekernel.StatfsOut)(m.Grow(int(unsafe.Sizeof(fusekernel.StatfsOut{}))))
out.St.Blocks = o.Blocks
out.St.Bfree = o.BlocksFree
out.St.Bavail = o.BlocksAvailable
......@@ -620,7 +620,7 @@ func (c *Connection) kernelResponseForOp(
out.St.Frsize = o.BlockSize
case *initOp:
out := (*fusekernel.InitOut)(m.Grow(unsafe.Sizeof(fusekernel.InitOut{})))
out := (*fusekernel.InitOut)(m.Grow(int(unsafe.Sizeof(fusekernel.InitOut{}))))
out.Major = o.Library.Major
out.Minor = o.Library.Minor
......
internal/buffer/out_message.go
View file @ 641629d1
......@@ -23,14 +23,9 @@ import (
"github.com/jacobsa/fuse/internal/fusekernel"
)
const outHeaderSize = unsafe.Sizeof(fusekernel.OutHeader{})
// OutMessage structs begin life with Len() == OutMessageInitialSize.
const OutMessageInitialSize = outHeaderSize
// We size out messages to be large enough to hold a header for the response
// plus the largest read that may come in.
const outMessageSize = outHeaderSize + MaxReadSize
// OutMessageHeaderSize is the size of the leading header in every
// properly-constructed OutMessage. Reset brings the message back to this size.
const OutMessageHeaderSize = int(unsafe.Sizeof(fusekernel.OutHeader{}))
// OutMessage provides a mechanism for constructing a single contiguous fuse
// message from multiple segments, where the first segment is always a
......@@ -38,72 +33,81 @@ const outMessageSize = outHeaderSize + MaxReadSize
//
// Must be initialized with Reset.
type OutMessage struct {
offset uintptr
storage [outMessageSize]byte
// The offset into payload to which we're currently writing.
payloadOffset int
header fusekernel.OutHeader
payload [MaxReadSize]byte
}
// Make sure alignment works out correctly, at least for the header.
// Make sure that the header and payload are contiguous.
func init() {
a := unsafe.Alignof(OutMessage{})
o := unsafe.Offsetof(OutMessage{}.storage)
e := unsafe.Alignof(fusekernel.OutHeader{})
a := unsafe.Offsetof(OutMessage{}.header) + uintptr(OutMessageHeaderSize)
b := unsafe.Offsetof(OutMessage{}.payload)
if a%e != 0 || o%e != 0 {
log.Panicf("Bad alignment or offset: %d, %d, need %d", a, o, e)
if a != b {
log.Panicf(
"header ends at offset %d, but payload starts at offset %d",
a, b)
}
}
// Reset the message so that it is ready to be used again. Afterward, the
// contents are solely a zeroed header.
// Reset resets m so that it's ready to be used again. Afterward, the contents
// are solely a zeroed fusekernel.OutHeader struct.
func (m *OutMessage) Reset() {
m.offset = OutMessageInitialSize
memclr(unsafe.Pointer(&m.storage), OutMessageInitialSize)
m.payloadOffset = 0
memclr(unsafe.Pointer(&m.header), uintptr(OutMessageHeaderSize))
}
// Return a pointer to the header at the start of the message.
func (b *OutMessage) OutHeader() (h *fusekernel.OutHeader) {
h = (*fusekernel.OutHeader)(unsafe.Pointer(&b.storage))
return
// OutHeader returns a pointer to the header at the start of the message.
func (m *OutMessage) OutHeader() *fusekernel.OutHeader {
return &m.header
}
// Grow the buffer by the supplied number of bytes, returning a pointer to the
// start of the new segment, which is zeroed. If there is no space left, return
// the nil pointer.
func (b *OutMessage) Grow(size uintptr) (p unsafe.Pointer) {
p = b.GrowNoZero(size)
// Grow grows m's buffer by the given number of bytes, returning a pointer to
// the start of the new segment, which is guaranteed to be zeroed. If there is
// insufficient space, it returns nil.
func (m *OutMessage) Grow(n int) (p unsafe.Pointer) {
p = m.GrowNoZero(n)
if p != nil {
memclr(p, size)
memclr(p, uintptr(n))
}
return
}
// Equivalent to Grow, except the new segment is not zeroed. Use with caution!
func (b *OutMessage) GrowNoZero(size uintptr) (p unsafe.Pointer) {
if outMessageSize-b.offset < size {
// GrowNoZero is equivalent to Grow, except the new segment is not zeroed. Use
// with caution!
func (m *OutMessage) GrowNoZero(n int) (p unsafe.Pointer) {
// Will we overflow the buffer?
o := m.payloadOffset
if len(m.payload)-o < n {
return
}
p = unsafe.Pointer(uintptr(unsafe.Pointer(&b.storage)) + b.offset)
b.offset += size
p = unsafe.Pointer(uintptr(unsafe.Pointer(&m.payload)) + uintptr(o))
m.payloadOffset = o + n
return
}
// Shrink to the supplied size. Panic if the size is greater than Len() or less
// than OutMessageInitialSize.
func (b *OutMessage) ShrinkTo(n uintptr) {
if n < OutMessageInitialSize || n > b.offset {
panic(fmt.Sprintf("ShrinkTo(%d) out of range for offset %d", n, b.offset))
// ShrinkTo shrinks m to the given size. It panics if the size is greater than
// Len() or less than OutMessageHeaderSize.
func (m *OutMessage) ShrinkTo(n int) {
if n < OutMessageHeaderSize || n > m.Len() {
panic(fmt.Sprintf(
"ShrinkTo(%d) out of range (current Len: %d)",
n,
m.Len()))
}
b.offset = n
m.payloadOffset = n - OutMessageHeaderSize
}
// Equivalent to growing by the length of p, then copying p over the new
// segment. Panics if there is not enough room available.
func (b *OutMessage) Append(src []byte) {
p := b.GrowNoZero(uintptr(len(src)))
// Append is equivalent to growing by len(src), then copying src over the new
// segment. Int panics if there is not enough room available.
func (m *OutMessage) Append(src []byte) {
p := m.GrowNoZero(len(src))
if p == nil {
panic(fmt.Sprintf("Can't grow %d bytes", len(src)))
}
......@@ -114,10 +118,9 @@ func (b *OutMessage) Append(src []byte) {
return
}
// Equivalent to growing by the length of s, then copying s over the new
// segment. Panics if there is not enough room available.
func (b *OutMessage) AppendString(src string) {
p := b.GrowNoZero(uintptr(len(src)))
// AppendString is like Append, but accepts string input.
func (m *OutMessage) AppendString(src string) {
p := m.GrowNoZero(len(src))
if p == nil {
panic(fmt.Sprintf("Can't grow %d bytes", len(src)))
}
......@@ -128,12 +131,20 @@ func (b *OutMessage) AppendString(src string) {
return
}
// Return the current size of the buffer.
func (b *OutMessage) Len() int {
return int(b.offset)
// Len returns the current size of the message, including the leading header.
func (m *OutMessage) Len() int {
return OutMessageHeaderSize + m.payloadOffset
}
// Return a reference to the current contents of the buffer.
func (b *OutMessage) Bytes() []byte {
return b.storage[:int(b.offset)]
// Bytes returns a reference to the current contents of the buffer, including
// the leading header.
func (m *OutMessage) Bytes() []byte {
l := m.Len()
sh := reflect.SliceHeader{
Data: uintptr(unsafe.Pointer(&m.header)),
Len: l,
Cap: l,
}
return *(*[]byte)(unsafe.Pointer(&sh))
}
internal/buffer/out_message_test.go
View file @ 641629d1
......@@ -101,7 +101,7 @@ func TestOutMessageAppend(t *testing.T) {
om.Append(wantPayload[4:])
// The result should be a zeroed header followed by the desired payload.
const wantLen = int(OutMessageInitialSize) + len(wantPayloadStr)
const wantLen = OutMessageHeaderSize + len(wantPayloadStr)
if got, want := om.Len(), wantLen; got != want {
t.Errorf("om.Len() = %d, want %d", got, want)
......@@ -113,7 +113,7 @@ func TestOutMessageAppend(t *testing.T) {
}
want := append(
make([]byte, OutMessageInitialSize),
make([]byte, OutMessageHeaderSize),
wantPayload...)
if !bytes.Equal(b, want) {
......@@ -131,7 +131,7 @@ func TestOutMessageAppendString(t *testing.T) {
om.AppendString(wantPayload[4:])
// The result should be a zeroed header followed by the desired payload.
const wantLen = int(OutMessageInitialSize) + len(wantPayload)
const wantLen = OutMessageHeaderSize + len(wantPayload)
if got, want := om.Len(), wantLen; got != want {
t.Errorf("om.Len() = %d, want %d", got, want)
......@@ -143,7 +143,7 @@ func TestOutMessageAppendString(t *testing.T) {
}
want := append(
make([]byte, OutMessageInitialSize),
make([]byte, OutMessageHeaderSize),
wantPayload...)
if !bytes.Equal(b, want) {
......@@ -159,10 +159,10 @@ func TestOutMessageShrinkTo(t *testing.T) {
om.AppendString("burrito")
// Shrink it.
om.ShrinkTo(OutMessageInitialSize + uintptr(len("taco")))
om.ShrinkTo(OutMessageHeaderSize + len("taco"))
// The result should be a zeroed header followed by "taco".
const wantLen = int(OutMessageInitialSize) + len("taco")
const wantLen = OutMessageHeaderSize + len("taco")
if got, want := om.Len(), wantLen; got != want {
t.Errorf("om.Len() = %d, want %d", got, want)
......@@ -174,7 +174,7 @@ func TestOutMessageShrinkTo(t *testing.T) {
}
want := append(
make([]byte, OutMessageInitialSize),
make([]byte, OutMessageHeaderSize),
"taco"...)
if !bytes.Equal(b, want) {
......@@ -233,7 +233,7 @@ func TestOutMessageReset(t *testing.T) {
om.Reset()
// Check that the length was updated.
if got, want := int(om.Len()), int(OutMessageInitialSize); got != want {
if got, want := om.Len(), OutMessageHeaderSize; got != want {
t.Fatalf("om.Len() = %d, want %d", got, want)
}
......@@ -269,7 +269,7 @@ func TestOutMessageGrow(t *testing.T) {
t.Fatalf("fillWithGarbage: %v", err)
}
om.ShrinkTo(OutMessageInitialSize)
om.ShrinkTo(OutMessageHeaderSize)
}
// Call Grow.
......@@ -278,7 +278,7 @@ func TestOutMessageGrow(t *testing.T) {
}
// Check the resulting length in two ways.
const wantLen = int(payloadSize + OutMessageInitialSize)
const wantLen = payloadSize + OutMessageHeaderSize
if got, want := om.Len(), wantLen; got != want {
t.Errorf("om.Len() = %d, want %d", got)
}
......@@ -289,7 +289,7 @@ func TestOutMessageGrow(t *testing.T) {
}
// Check that the payload was zeroed.
for i, x := range b[OutMessageInitialSize:] {
for i, x := range b[OutMessageHeaderSize:] {
if x != 0 {
t.Fatalf("non-zero byte 0x%02x at payload offset %d", x, i)
}
......@@ -304,7 +304,7 @@ func BenchmarkOutMessageReset(b *testing.B) {
om.Reset()
}
b.SetBytes(int64(unsafe.Offsetof(om.storage)) + int64(om.offset))
b.SetBytes(int64(unsafe.Offsetof(om.payload)))
})
// Many megabytes worth of buffers, which should defeat the CPU cache.
......@@ -321,7 +321,7 @@ func BenchmarkOutMessageReset(b *testing.B) {
oms[i%numMessages].Reset()
}
b.SetBytes(int64(unsafe.Offsetof(oms[0].storage)) + int64(oms[0].offset))
b.SetBytes(int64(unsafe.Offsetof(oms[0].payload)))
})
}
......@@ -331,7 +331,7 @@ func BenchmarkOutMessageGrowShrink(b *testing.B) {
var om OutMessage
for i := 0; i < b.N; i++ {
om.Grow(MaxReadSize)
om.ShrinkTo(OutMessageInitialSize)
om.ShrinkTo(OutMessageHeaderSize)
}
b.SetBytes(int64(MaxReadSize))
......@@ -349,7 +349,7 @@ func BenchmarkOutMessageGrowShrink(b *testing.B) {
for i := 0; i < b.N; i++ {
oms[i%numMessages].Grow(MaxReadSize)
oms[i%numMessages].ShrinkTo(OutMessageInitialSize)
oms[i%numMessages].ShrinkTo(OutMessageHeaderSize)
}
b.SetBytes(int64(MaxReadSize))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7