Commit dacbb6cf authored by Kirill Smelkov's avatar Kirill Smelkov

.

parent 5f440f01
// DO NOT EDIT - AUTOGENERATED (by gen-fsbtree from github.com/cznic/b bcff30a) // DO NOT EDIT - AUTOGENERATED (by gen-fsbtree from github.com/cznic/b bcff30a)
// KEY=zodb.Oid VALUE=zodb.Tid // KEY=zodb.Oid VALUE=int64
// ---- 8< ---- // ---- 8< ----
// Copyright 2014 The b Authors. All rights reserved. // Copyright 2014 The b Authors. All rights reserved.
...@@ -72,7 +72,7 @@ type ( ...@@ -72,7 +72,7 @@ type (
de struct { // d element de struct { // d element
k zodb.Oid k zodb.Oid
v zodb.Tid v int64
} }
// Enumerator captures the state of enumerating a tree. It is returned // Enumerator captures the state of enumerating a tree. It is returned
...@@ -338,7 +338,7 @@ func (t *Tree) Delete(k zodb.Oid) (ok bool) { ...@@ -338,7 +338,7 @@ func (t *Tree) Delete(k zodb.Oid) (ok bool) {
} }
} }
func (t *Tree) extract(q *d, i int) { // (r zodb.Tid) { func (t *Tree) extract(q *d, i int) { // (r int64) {
t.ver++ t.ver++
//r = q.d[i].v // prepared for Extract //r = q.d[i].v // prepared for Extract
q.c-- q.c--
...@@ -388,7 +388,7 @@ func (t *Tree) find(q interface{}, k zodb.Oid) (i int, ok bool) { ...@@ -388,7 +388,7 @@ func (t *Tree) find(q interface{}, k zodb.Oid) (i int, ok bool) {
// First returns the first item of the tree in the key collating order, or // First returns the first item of the tree in the key collating order, or
// (zero-value, zero-value) if the tree is empty. // (zero-value, zero-value) if the tree is empty.
func (t *Tree) First() (k zodb.Oid, v zodb.Tid) { func (t *Tree) First() (k zodb.Oid, v int64) {
if q := t.first; q != nil { if q := t.first; q != nil {
q := &q.d[0] q := &q.d[0]
k, v = q.k, q.v k, v = q.k, q.v
...@@ -398,7 +398,7 @@ func (t *Tree) First() (k zodb.Oid, v zodb.Tid) { ...@@ -398,7 +398,7 @@ func (t *Tree) First() (k zodb.Oid, v zodb.Tid) {
// Get returns the value associated with k and true if it exists. Otherwise Get // Get returns the value associated with k and true if it exists. Otherwise Get
// returns (zero-value, false). // returns (zero-value, false).
func (t *Tree) Get(k zodb.Oid) (v zodb.Tid, ok bool) { func (t *Tree) Get(k zodb.Oid) (v int64, ok bool) {
q := t.r q := t.r
if q == nil { if q == nil {
return return
...@@ -424,7 +424,7 @@ func (t *Tree) Get(k zodb.Oid) (v zodb.Tid, ok bool) { ...@@ -424,7 +424,7 @@ func (t *Tree) Get(k zodb.Oid) (v zodb.Tid, ok bool) {
} }
} }
func (t *Tree) insert(q *d, i int, k zodb.Oid, v zodb.Tid) *d { func (t *Tree) insert(q *d, i int, k zodb.Oid, v int64) *d {
t.ver++ t.ver++
c := q.c c := q.c
if i < c { if i < c {
...@@ -439,7 +439,7 @@ func (t *Tree) insert(q *d, i int, k zodb.Oid, v zodb.Tid) *d { ...@@ -439,7 +439,7 @@ func (t *Tree) insert(q *d, i int, k zodb.Oid, v zodb.Tid) *d {
// Last returns the last item of the tree in the key collating order, or // Last returns the last item of the tree in the key collating order, or
// (zero-value, zero-value) if the tree is empty. // (zero-value, zero-value) if the tree is empty.
func (t *Tree) Last() (k zodb.Oid, v zodb.Tid) { func (t *Tree) Last() (k zodb.Oid, v int64) {
if q := t.last; q != nil { if q := t.last; q != nil {
q := &q.d[q.c-1] q := &q.d[q.c-1]
k, v = q.k, q.v k, v = q.k, q.v
...@@ -452,7 +452,7 @@ func (t *Tree) Len() int { ...@@ -452,7 +452,7 @@ func (t *Tree) Len() int {
return t.c return t.c
} }
func (t *Tree) overflow(p *x, q *d, pi, i int, k zodb.Oid, v zodb.Tid) { func (t *Tree) overflow(p *x, q *d, pi, i int, k zodb.Oid, v int64) {
t.ver++ t.ver++
l, r := p.siblings(pi) l, r := p.siblings(pi)
...@@ -533,7 +533,7 @@ func (t *Tree) SeekLast() (e *Enumerator, err error) { ...@@ -533,7 +533,7 @@ func (t *Tree) SeekLast() (e *Enumerator, err error) {
} }
// Set sets the value associated with k. // Set sets the value associated with k.
func (t *Tree) Set(k zodb.Oid, v zodb.Tid) { func (t *Tree) Set(k zodb.Oid, v int64) {
//dbg("--- PRE Set(%v, %v)\n%s", k, v, t.dump()) //dbg("--- PRE Set(%v, %v)\n%s", k, v, t.dump())
//defer func() { //defer func() {
// dbg("--- POST\n%s\n====\n", t.dump()) // dbg("--- POST\n%s\n====\n", t.dump())
...@@ -598,11 +598,11 @@ func (t *Tree) Set(k zodb.Oid, v zodb.Tid) { ...@@ -598,11 +598,11 @@ func (t *Tree) Set(k zodb.Oid, v zodb.Tid) {
// tree.Put(k, func(zodb.Oid, bool){ return v, true }) // tree.Put(k, func(zodb.Oid, bool){ return v, true })
// //
// modulo the differing return values. // modulo the differing return values.
func (t *Tree) Put(k zodb.Oid, upd func(oldV zodb.Tid, exists bool) (newV zodb.Tid, write bool)) (oldV zodb.Tid, written bool) { func (t *Tree) Put(k zodb.Oid, upd func(oldV int64, exists bool) (newV int64, write bool)) (oldV int64, written bool) {
pi := -1 pi := -1
var p *x var p *x
q := t.r q := t.r
var newV zodb.Tid var newV int64
if q == nil { if q == nil {
// new KV pair in empty tree // new KV pair in empty tree
newV, written = upd(newV, false) newV, written = upd(newV, false)
...@@ -664,7 +664,7 @@ func (t *Tree) Put(k zodb.Oid, upd func(oldV zodb.Tid, exists bool) (newV zodb.T ...@@ -664,7 +664,7 @@ func (t *Tree) Put(k zodb.Oid, upd func(oldV zodb.Tid, exists bool) (newV zodb.T
} }
} }
func (t *Tree) split(p *x, q *d, pi, i int, k zodb.Oid, v zodb.Tid) { func (t *Tree) split(p *x, q *d, pi, i int, k zodb.Oid, v int64) {
t.ver++ t.ver++
r := btDPool.Get().(*d) r := btDPool.Get().(*d)
if q.n != nil { if q.n != nil {
...@@ -826,7 +826,7 @@ func (e *Enumerator) Close() { ...@@ -826,7 +826,7 @@ func (e *Enumerator) Close() {
// Next returns the currently enumerated item, if it exists and moves to the // Next returns the currently enumerated item, if it exists and moves to the
// next item in the key collation order. If there is no item to return, err == // next item in the key collation order. If there is no item to return, err ==
// io.EOF is returned. // io.EOF is returned.
func (e *Enumerator) Next() (k zodb.Oid, v zodb.Tid, err error) { func (e *Enumerator) Next() (k zodb.Oid, v int64, err error) {
if err = e.err; err != nil { if err = e.err; err != nil {
return return
} }
...@@ -880,7 +880,7 @@ func (e *Enumerator) next() error { ...@@ -880,7 +880,7 @@ func (e *Enumerator) next() error {
// Prev returns the currently enumerated item, if it exists and moves to the // Prev returns the currently enumerated item, if it exists and moves to the
// previous item in the key collation order. If there is no item to return, err // previous item in the key collation order. If there is no item to return, err
// == io.EOF is returned. // == io.EOF is returned.
func (e *Enumerator) Prev() (k zodb.Oid, v zodb.Tid, err error) { func (e *Enumerator) Prev() (k zodb.Oid, v int64, err error) {
if err = e.err; err != nil { if err = e.err; err != nil {
return return
} }
......
#!/bin/sh -e #!/bin/sh -e
# generate b.Tree with compile-time KEY=zodb.Oid, VALUE=zodb.Tid # generate b.Tree with compile-time KEY=zodb.Oid, VALUE=int64
KEY=zodb.Oid KEY=zodb.Oid
VALUE=zodb.Tid VALUE=int64
b=github.com/cznic/b b=github.com/cznic/b
Bdir=`go list -f '{{.Dir}}' $b` Bdir=`go list -f '{{.Dir}}' $b`
......
...@@ -36,25 +36,15 @@ type fsIndex struct { ...@@ -36,25 +36,15 @@ type fsIndex struct {
} }
// TODO Encode (__getstate__) // TODO Encode (__getstate__) ?
// TODO Decode (__setstate__) // TODO Decode (__setstate__) ?
// TODO ? save(pos, fname)
// TODO ? klass.load(fname)
/*
// __getitem__
func (fsi *fsIndex) Get(oid zodb.Oid) zodb.Tid { // XXX ,ok ?
tid, ok := fsi.oid2tid.Get(oid)
return tid, ok
}
*/
// __setitem__ -> Set // NOTE Get/Set/... are taken as-is from fsb.Tree
// on-disk index format // on-disk index format
// (changed in 2010 in https://github.com/zopefoundation/ZODB/commit/1bb14faf) // (changed in 2010 in https://github.com/zopefoundation/ZODB/commit/1bb14faf)
// //
// pos (?) (int or long - INT_TYPES) // topPos position pointing just past the last committed transaction
// (oid[:6], v.toString) # ._data[oid]-> v (fsBucket().toString()) // (oid[:6], v.toString) # ._data[oid]-> v (fsBucket().toString())
// (oid[:6], v.toString) // (oid[:6], v.toString)
// ... // ...
...@@ -62,51 +52,61 @@ func (fsi *fsIndex) Get(oid zodb.Oid) zodb.Tid { // XXX ,ok ? ...@@ -62,51 +52,61 @@ func (fsi *fsIndex) Get(oid zodb.Oid) zodb.Tid { // XXX ,ok ?
// //
// //
// fsBucket.toString(): // fsBucket.toString():
// oid[6:8]oid[6:8]oid[6:8]...tid[0:6]tid[0:6]tid[0:6]... // oid[6:8]oid[6:8]oid[6:8]...pos[0:6]pos[0:6]pos[0:6]...
const oidPrefixMask zodb.Oid = ((1<<64-1) ^ (1<<16 -1)) // 0xffffffffffff0000 const oidPrefixMask zodb.Oid = ((1<<64-1) ^ (1<<16 -1)) // 0xffffffffffff0000
// IndexIOError is the error type returned by index save and load routines
type IndexIOError struct {
Op string // operation performed - "save" or "load"
Err error // error that occured during the operation
}
func (e *IndexIOError) Error() string {
s := "index " + e.Op + ": " + e.Err.Error()
return s
}
// Save saves the index to a writer // Save saves the index to a writer
// XXX proper error context func (fsi *fsIndex) Save(topPos int64, w io.Writer) error {
func (fsi *fsIndex) Save(fsPos int64, w io.Writer) error {
p := pickle.NewEncoder(w) p := pickle.NewEncoder(w)
err := p.Encode(fsPos) err := p.Encode(topPos)
if err != nil { if err != nil {
return err goto out
} }
var oidb [8]byte var oidb [8]byte
var tidb [8]byte var posb [8]byte
var oidPrefixCur zodb.Oid // current oid[0:6] with [6:8] = 00 var oidPrefixCur zodb.Oid // current oid[0:6] with [6:8] = 00
oidBuf := []byte{} // current oid[6:8]oid[6:8]... oidBuf := []byte{} // current oid[6:8]oid[6:8]...
tidBuf := []byte{} // current tid[0:6]tid[0:6]... posBuf := []byte{} // current pos[0:6]pos[0:6]...
var t [2]interface{} // tuple for (oid, fsBucket.toString()) var t [2]interface{} // tuple for (oid, fsBucket.toString())
e, err := fsi.SeekFirst() e, err := fsi.SeekFirst()
if err == io.EOF { if err == io.EOF { // always only io.EOF indicating an empty btree
goto skip // empty btree goto skip
} }
for { for {
oid, tid, errStop := e.Next() oid, pos, errStop := e.Next()
oidPrefix := oid & oidPrefixMask oidPrefix := oid & oidPrefixMask
if oidPrefix != oidPrefixCur { if oidPrefix != oidPrefixCur {
// emit pickle for current oid06 // emit (oid[:6], oid[6:8]oid[6:8]...pos[0:6]pos[0:6]...)
binary.BigEndian.PutUint64(oidb[:], uint64(oid)) binary.BigEndian.PutUint64(oidb[:], uint64(oid))
t[0] = oidb[0:6] t[0] = oidb[0:6]
t[1] = bytes.Join([][]byte{oidBuf, tidBuf}, nil) t[1] = bytes.Join([][]byte{oidBuf, posBuf}, nil)
err = p.Encode(t) err = p.Encode(t)
if err != nil { if err != nil {
return err goto out
} }
oidPrefixCur = oidPrefix oidPrefixCur = oidPrefix
oidBuf = oidBuf[:0] oidBuf = oidBuf[:0]
tidBuf = tidBuf[:0] posBuf = posBuf[:0]
} }
if errStop != nil { if errStop != nil {
...@@ -114,21 +114,34 @@ func (fsi *fsIndex) Save(fsPos int64, w io.Writer) error { ...@@ -114,21 +114,34 @@ func (fsi *fsIndex) Save(fsPos int64, w io.Writer) error {
} }
binary.BigEndian.PutUint64(oidb[:], uint64(oid)) binary.BigEndian.PutUint64(oidb[:], uint64(oid))
binary.BigEndian.PutUint64(tidb[:], uint64(tid)) binary.BigEndian.PutUint64(posb[:], uint64(pos))
// XXX check pos does not overflow 6 bytes
oidBuf = append(oidBuf, oidb[6:8]...) oidBuf = append(oidBuf, oidb[6:8]...)
tidBuf = append(tidBuf, tidb[0:6]...) posBuf = append(posBuf, posb[0:6]...)
} }
e.Close() e.Close()
skip: skip:
err = p.Encode(pickle.None{}) err = p.Encode(pickle.None{})
return err
out:
if err == nil {
return err
}
if _, ok := err.(pikle.TypeError); ok {
panic(err) // all our types are expected to be supported by pickle
}
// otherwise it is an error returned by writer, which should already
// have filename & op as context.
return &IndexIOError{"save", err}
} }
// LoadIndex loads index from a reader // LoadIndex loads index from a reader
func LoadIndex(r io.Reader) (fsPos int64, fsi *fsIndex, err error) { func LoadIndex(r io.Reader) (topPos int64, fsi *fsIndex, err error) {
p := pickle.NewDecoder(r) p := pickle.NewDecoder(r)
// if we can know file position we can show it in error context // if we can know file position we can show it in error context
...@@ -140,18 +153,18 @@ func LoadIndex(r io.Reader) (fsPos int64, fsi *fsIndex, err error) { ...@@ -140,18 +153,18 @@ func LoadIndex(r io.Reader) (fsPos int64, fsi *fsIndex, err error) {
} }
} }
xpos, err := p.Decode() xtopPos, err := p.Decode()
if err != nil { if err != nil {
// TODO err // TODO err
} }
fsPos, ok := xpos.(int64) topPos, ok := xtopPos.(int64)
if !ok { if !ok {
// TODO err // TODO err
} }
fsi = &fsIndex{} // TODO cmpFunc ... fsi = &fsIndex{} // TODO cmpFunc ...
var oidb [8]byte var oidb [8]byte
var tidb [8]byte var posb [8]byte
loop: loop:
for { for {
...@@ -194,22 +207,61 @@ loop: ...@@ -194,22 +207,61 @@ loop:
n := len(kvBuf) / 8 n := len(kvBuf) / 8
oidBuf := kvBuf[:n*2] oidBuf := kvBuf[:n*2]
tidBuf := kvBuf[n*2:] posBuf := kvBuf[n*2:]
for i:=0; i<n; i++ { for i:=0; i<n; i++ {
oid := zodb.Oid(binary.BigEndian.Uint16(oidBuf[i*2:])) oid := zodb.Oid(binary.BigEndian.Uint16(oidBuf[i*2:]))
oid |= oidPrefix oid |= oidPrefix
copy(tidb[2:], tidBuf[i*6:]) copy(posb[2:], posBuf[i*6:])
tid := zodb.Tid(binary.BigEndian.Uint64(tidb[:])) tid := zodb.Tid(binary.BigEndian.Uint64(posb[:]))
fsi.Set(oid, tid) fsi.Set(oid, tid)
} }
} }
return fsPos, fsi, nil return topPos, fsi, nil
out:
if err == nil {
return topPos, fsi, err
}
rname := IOName(r)
xerror:
// same for file name // same for file name
rname, _ := r.(interface{ Name() string }) rname, _ := r.(interface{ Name() string })
} }
// IOName returns a "filename" associated with io.Reader, io.Writer, net.Conn, ...
// if name cannot be deterined - "" is returned.
func IOName(f interface {}) string {
switch f := f.(type) {
case *os.File:
// XXX better via interface { Name() string } ?
// but Name() is too broad compared to FileName()
return f.Name()
case net.Conn:
// XXX not including LocalAddr is ok?
return f.RemoteAddr.String()
case *io.LimitedReader:
return IOName(f.R)
case *io.PipeReader:
fallthrough
case *io.PipeWriter:
return "pipe"
// XXX SectionReader MultiReader TeeReader
// bufio.Reader bufio.Writer bufio.Scanner
default:
return ""
}
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment