Skip to content

B
BTrees
Commit 703fc7e7 authored by Tres Seaver's avatar Tres Seaver
Browse files
Options

Copy trunk's BTrees package.

parent 3d003537
Hide whitespace changes
Inline Side-by-side
Showing with 13759 additions and 0 deletions
BTrees/BTreeItemsTemplate.c 0 → 100644
View file @ 703fc7e7
/*****************************************************************************
Copyright (c) 2001, 2002 Zope Foundation and Contributors.
All Rights Reserved.
This software is subject to the provisions of the Zope Public License,
Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
****************************************************************************/
#define BTREEITEMSTEMPLATE_C "$Id$\n"
/* A BTreeItems struct is returned from calling .items(), .keys() or
* .values() on a BTree-based data structure, and is also the result of
* taking slices of those. It represents a contiguous slice of a BTree.
*
* The start of the slice is in firstbucket, at offset first. The end of
* the slice is in lastbucket, at offset last. Both endpoints are inclusive.
* It must possible to get from firstbucket to lastbucket via following
* bucket 'next' pointers zero or more times. firstbucket, first, lastbucket,
* and last are readonly after initialization. An empty slice is represented
* by firstbucket == lastbucket == currentbucket == NULL.
*
* 'kind' determines whether this slice represents 'k'eys alone, 'v'alues
* alone, or 'i'items (key+value pairs). 'kind' is also readonly after
* initialization.
*
* The combination of currentbucket, currentoffset and pseudoindex acts as
* a search finger. Offset currentoffset in bucket currentbucket is at index
* pseudoindex, where pseudoindex==0 corresponds to offset first in bucket
* firstbucket, and pseudoindex==-1 corresponds to offset last in bucket
* lastbucket. The function BTreeItems_seek() can be used to set this combo
* correctly for any in-bounds index, and uses this combo on input to avoid
* needing to search from the start (or end) on each call. Calling
* BTreeItems_seek() with consecutive larger positions is very efficent.
* Calling it with consecutive smaller positions is more efficient than if
* a search finger weren't being used at all, but is still quadratic time
* in the number of buckets in the slice.
*/
typedef struct {
PyObject_HEAD
Bucket *firstbucket; /* First bucket */
Bucket *currentbucket; /* Current bucket (search finger) */
Bucket *lastbucket; /* Last bucket */
int currentoffset; /* Offset in currentbucket */
int pseudoindex; /* search finger index */
int first; /* Start offset in firstbucket */
int last; /* End offset in lastbucket */
char kind; /* 'k', 'v', 'i' */
} BTreeItems;
#define ITEMS(O)((BTreeItems*)(O))
static PyObject *
newBTreeItems(char kind,
Bucket *lowbucket, int lowoffset,
Bucket *highbucket, int highoffset);
static void
BTreeItems_dealloc(BTreeItems *self)
{
Py_XDECREF(self->firstbucket);
Py_XDECREF(self->lastbucket);
Py_XDECREF(self->currentbucket);
PyObject_DEL(self);
}
static Py_ssize_t
BTreeItems_length_or_nonzero(BTreeItems *self, int nonzero)
{
Py_ssize_t r;
Bucket *b, *next;
b = self->firstbucket;
if (b == NULL)
return 0;
r = self->last + 1 - self->first;
if (nonzero && r > 0)
/* Short-circuit if all we care about is nonempty */
return 1;
if (b == self->lastbucket)
return r;
Py_INCREF(b);
PER_USE_OR_RETURN(b, -1);
while ((next = b->next)) {
r += b->len;
if (nonzero && r > 0)
/* Short-circuit if all we care about is nonempty */
break;
if (next == self->lastbucket)
break; /* we already counted the last bucket */
Py_INCREF(next);
PER_UNUSE(b);
Py_DECREF(b);
b = next;
PER_USE_OR_RETURN(b, -1);
}
PER_UNUSE(b);
Py_DECREF(b);
return r >= 0 ? r : 0;
}
static Py_ssize_t
BTreeItems_length(BTreeItems *self)
{
return BTreeItems_length_or_nonzero(self, 0);
}
/*
** BTreeItems_seek
**
** Find the ith position in the BTreeItems.
**
** Arguments: self The BTree
** i the index to seek to, in 0 .. len(self)-1, or in
** -len(self) .. -1, as for indexing a Python sequence.
**
**
** Returns 0 if successful, -1 on failure to seek (like out-of-bounds).
** Upon successful return, index i is at offset self->currentoffset in bucket
** self->currentbucket.
*/
static int
BTreeItems_seek(BTreeItems *self, Py_ssize_t i)
{
int delta, pseudoindex, currentoffset;
Bucket *b, *currentbucket;
int error;
pseudoindex = self->pseudoindex;
currentoffset = self->currentoffset;
currentbucket = self->currentbucket;
if (currentbucket == NULL) goto no_match;
delta = i - pseudoindex;
while (delta > 0) { /* move right */
int max;
/* Want to move right delta positions; the most we can move right in
* this bucket is currentbucket->len - currentoffset - 1 positions.
*/
PER_USE_OR_RETURN(currentbucket, -1);
max = currentbucket->len - currentoffset - 1;
b = currentbucket->next;
PER_UNUSE(currentbucket);
if (delta <= max) {
currentoffset += delta;
pseudoindex += delta;
if (currentbucket == self->lastbucket
&& currentoffset > self->last) goto no_match;
break;
}
/* Move to start of next bucket. */
if (currentbucket == self->lastbucket || b == NULL) goto no_match;
currentbucket = b;
pseudoindex += max + 1;
delta -= max + 1;
currentoffset = 0;
}
while (delta < 0) { /* move left */
int status;
/* Want to move left -delta positions; the most we can move left in
* this bucket is currentoffset positions.
*/
if ((-delta) <= currentoffset) {
currentoffset += delta;
pseudoindex += delta;
if (currentbucket == self->firstbucket
&& currentoffset < self->first) goto no_match;
break;
}
/* Move to end of previous bucket. */
if (currentbucket == self->firstbucket) goto no_match;
status = PreviousBucket(&currentbucket, self->firstbucket);
if (status == 0)
goto no_match;
else if (status < 0)
return -1;
pseudoindex -= currentoffset + 1;
delta += currentoffset + 1;
PER_USE_OR_RETURN(currentbucket, -1);
currentoffset = currentbucket->len - 1;
PER_UNUSE(currentbucket);
}
assert(pseudoindex == i);
/* Alas, the user may have mutated the bucket since the last time we
* were called, and if they deleted stuff, we may be pointing into
* trash memory now.
*/
PER_USE_OR_RETURN(currentbucket, -1);
error = currentoffset < 0 || currentoffset >= currentbucket->len;
PER_UNUSE(currentbucket);
if (error) {
PyErr_SetString(PyExc_RuntimeError,
"the bucket being iterated changed size");
return -1;
}
Py_INCREF(currentbucket);
Py_DECREF(self->currentbucket);
self->currentbucket = currentbucket;
self->currentoffset = currentoffset;
self->pseudoindex = pseudoindex;
return 0;
no_match:
IndexError(i);
return -1;
}
/* Return the right kind ('k','v','i') of entry from bucket b at offset i.
* b must be activated. Returns NULL on error.
*/
static PyObject *
getBucketEntry(Bucket *b, int i, char kind)
{
PyObject *result = NULL;
assert(b);
assert(0 <= i && i < b->len);
switch (kind) {
case 'k':
COPY_KEY_TO_OBJECT(result, b->keys[i]);
break;
case 'v':
COPY_VALUE_TO_OBJECT(result, b->values[i]);
break;
case 'i': {
PyObject *key;
PyObject *value;;
COPY_KEY_TO_OBJECT(key, b->keys[i]);
if (!key) break;
COPY_VALUE_TO_OBJECT(value, b->values[i]);
if (!value) {
Py_DECREF(key);
break;
}
result = PyTuple_New(2);
if (result) {
PyTuple_SET_ITEM(result, 0, key);
PyTuple_SET_ITEM(result, 1, value);
}
else {
Py_DECREF(key);
Py_DECREF(value);
}
break;
}
default:
PyErr_SetString(PyExc_AssertionError,
"getBucketEntry: unknown kind");
break;
}
return result;
}
/*
** BTreeItems_item
**
** Arguments: self a BTreeItems structure
** i Which item to inspect
**
** Returns: the BTreeItems_item_BTree of self->kind, i
** (ie pulls the ith item out)
*/
static PyObject *
BTreeItems_item(BTreeItems *self, Py_ssize_t i)
{
PyObject *result;
if (BTreeItems_seek(self, i) < 0) return NULL;
PER_USE_OR_RETURN(self->currentbucket, NULL);
result = getBucketEntry(self->currentbucket, self->currentoffset,
self->kind);
PER_UNUSE(self->currentbucket);
return result;
}
/*
** BTreeItems_slice
**
** Creates a new BTreeItems structure representing the slice
** between the low and high range
**
** Arguments: self The old BTreeItems structure
** ilow The start index
** ihigh The end index
**
** Returns: BTreeItems item
*/
static PyObject *
BTreeItems_slice(BTreeItems *self, Py_ssize_t ilow, Py_ssize_t ihigh)
{
Bucket *lowbucket;
Bucket *highbucket;
int lowoffset;
int highoffset;
Py_ssize_t length = -1; /* len(self), but computed only if needed */
/* Complications:
* A Python slice never raises IndexError, but BTreeItems_seek does.
* Python did only part of index normalization before calling this:
* ilow may be < 0 now, and ihigh may be arbitrarily large. It's
* our responsibility to clip them.
* A Python slice is exclusive of the high index, but a BTreeItems
* struct is inclusive on both ends.
*/
/* First adjust ilow and ihigh to be legit endpoints in the Python
* sense (ilow inclusive, ihigh exclusive). This block duplicates the
* logic from Python's list_slice function (slicing for builtin lists).
*/
if (ilow < 0)
ilow = 0;
else {
if (length < 0)
length = BTreeItems_length(self);
if (ilow > length)
ilow = length;
}
if (ihigh < ilow)
ihigh = ilow;
else {
if (length < 0)
length = BTreeItems_length(self);
if (ihigh > length)
ihigh = length;
}
assert(0 <= ilow && ilow <= ihigh);
assert(length < 0 || ihigh <= length);
/* Now adjust for that our struct is inclusive on both ends. This is
* easy *except* when the slice is empty: there's no good way to spell
* that in an inclusive-on-both-ends scheme. For example, if the
* slice is btree.items([:0]), ilow == ihigh == 0 at this point, and if
* we were to subtract 1 from ihigh that would get interpreted by
* BTreeItems_seek as meaning the *entire* set of items. Setting ilow==1
* and ihigh==0 doesn't work either, as BTreeItems_seek raises IndexError
* if we attempt to seek to ilow==1 when the underlying sequence is empty.
* It seems simplest to deal with empty slices as a special case here.
*/
if (ilow == ihigh) {
/* empty slice */
lowbucket = highbucket = NULL;
lowoffset = 1;
highoffset = 0;
}
else {
assert(ilow < ihigh);
--ihigh; /* exclusive -> inclusive */
if (BTreeItems_seek(self, ilow) < 0) return NULL;
lowbucket = self->currentbucket;
lowoffset = self->currentoffset;
if (BTreeItems_seek(self, ihigh) < 0) return NULL;
highbucket = self->currentbucket;
highoffset = self->currentoffset;
}
return newBTreeItems(self->kind,
lowbucket, lowoffset, highbucket, highoffset);
}
static PySequenceMethods BTreeItems_as_sequence = {
(lenfunc) BTreeItems_length,
(binaryfunc)0,
(ssizeargfunc)0,
(ssizeargfunc) BTreeItems_item,
(ssizessizeargfunc) BTreeItems_slice,
};
/* Number Method items (just for nb_nonzero!) */
static int
BTreeItems_nonzero(BTreeItems *self)
{
return BTreeItems_length_or_nonzero(self, 1);
}
static PyNumberMethods BTreeItems_as_number_for_nonzero = {
0,0,0,0,0,0,0,0,0,0,
(inquiry)BTreeItems_nonzero};
static PyTypeObject BTreeItemsType = {
PyObject_HEAD_INIT(NULL)
0, /*ob_size*/
MOD_NAME_PREFIX "BTreeItems", /*tp_name*/
sizeof(BTreeItems), /*tp_basicsize*/
0, /*tp_itemsize*/
/* methods */
(destructor) BTreeItems_dealloc, /*tp_dealloc*/
(printfunc)0, /*tp_print*/
(getattrfunc)0, /*obsolete tp_getattr*/
(setattrfunc)0, /*obsolete tp_setattr*/
(cmpfunc)0, /*tp_compare*/
(reprfunc)0, /*tp_repr*/
&BTreeItems_as_number_for_nonzero, /*tp_as_number*/
&BTreeItems_as_sequence, /*tp_as_sequence*/
0, /*tp_as_mapping*/
(hashfunc)0, /*tp_hash*/
(ternaryfunc)0, /*tp_call*/
(reprfunc)0, /*tp_str*/
0, /*tp_getattro*/
0, /*tp_setattro*/
/* Space for future expansion */
0L,0L,
"Sequence type used to iterate over BTree items." /* Documentation string */
};
/* Returns a new BTreeItems object representing the contiguous slice from
* offset lowoffset in bucket lowbucket through offset highoffset in bucket
* highbucket, inclusive. Pass lowbucket == NULL for an empty slice.
* The currentbucket is set to lowbucket, currentoffset ot lowoffset, and
* pseudoindex to 0. kind is 'k', 'v' or 'i' (see BTreeItems struct docs).
*/
static PyObject *
newBTreeItems(char kind,
Bucket *lowbucket, int lowoffset,
Bucket *highbucket, int highoffset)
{
BTreeItems *self;
UNLESS (self = PyObject_NEW(BTreeItems, &BTreeItemsType)) return NULL;
self->kind=kind;
self->first=lowoffset;
self->last=highoffset;
if (! lowbucket || ! highbucket
|| (lowbucket == highbucket && lowoffset > highoffset))
{
self->firstbucket = 0;
self->lastbucket = 0;
self->currentbucket = 0;
}
else
{
Py_INCREF(lowbucket);
self->firstbucket = lowbucket;
Py_INCREF(highbucket);
self->lastbucket = highbucket;
Py_INCREF(lowbucket);
self->currentbucket = lowbucket;
}
self->currentoffset = lowoffset;
self->pseudoindex = 0;
return OBJECT(self);
}
static int
nextBTreeItems(SetIteration *i)
{
if (i->position >= 0)
{
if (i->position)
{
DECREF_KEY(i->key);
DECREF_VALUE(i->value);
}
if (BTreeItems_seek(ITEMS(i->set), i->position) >= 0)
{
Bucket *currentbucket;
currentbucket = BUCKET(ITEMS(i->set)->currentbucket);
UNLESS(PER_USE(currentbucket))
{
/* Mark iteration terminated, so that finiSetIteration doesn't
* try to redundantly decref the key and value
*/
i->position = -1;
return -1;
}
COPY_KEY(i->key, currentbucket->keys[ITEMS(i->set)->currentoffset]);
INCREF_KEY(i->key);
COPY_VALUE(i->value,
currentbucket->values[ITEMS(i->set)->currentoffset]);
INCREF_VALUE(i->value);
i->position ++;
PER_UNUSE(currentbucket);
}
else
{
i->position = -1;
PyErr_Clear();
}
}
return 0;
}
static int
nextTreeSetItems(SetIteration *i)
{
if (i->position >= 0)
{
if (i->position)
{
DECREF_KEY(i->key);
}
if (BTreeItems_seek(ITEMS(i->set), i->position) >= 0)
{
Bucket *currentbucket;
currentbucket = BUCKET(ITEMS(i->set)->currentbucket);
UNLESS(PER_USE(currentbucket))
{
/* Mark iteration terminated, so that finiSetIteration doesn't
* try to redundantly decref the key and value
*/
i->position = -1;
return -1;
}
COPY_KEY(i->key, currentbucket->keys[ITEMS(i->set)->currentoffset]);
INCREF_KEY(i->key);
i->position ++;
PER_UNUSE(currentbucket);
}
else
{
i->position = -1;
PyErr_Clear();
}
}
return 0;
}
/* Support for the iteration protocol new in Python 2.2. */
static PyTypeObject BTreeIter_Type;
/* The type of iterator objects, returned by e.g. iter(IIBTree()). */
typedef struct {
PyObject_HEAD
/* We use a BTreeItems object because it's convenient and flexible.
* We abuse it two ways:
* 1. We set currentbucket to NULL when the iteration is finished.
* 2. We don't bother keeping pseudoindex in synch.
*/
BTreeItems *pitems;
} BTreeIter;
/* Return a new iterator object, to traverse the keys and/or values
* represented by pitems. pitems must not be NULL. Returns NULL if error.
*/
static BTreeIter *
BTreeIter_new(BTreeItems *pitems)
{
BTreeIter *result;
assert(pitems != NULL);
result = PyObject_New(BTreeIter, &BTreeIter_Type);
if (result) {
Py_INCREF(pitems);
result->pitems = pitems;
}
return result;
}
/* The iterator's tp_dealloc slot. */
static void
BTreeIter_dealloc(BTreeIter *bi)
{
Py_DECREF(bi->pitems);
PyObject_Del(bi);
}
/* The implementation of the iterator's tp_iternext slot. Returns "the next"
* item; returns NULL if error; returns NULL without setting an error if the
* iteration is exhausted (that's the way to terminate the iteration protocol).
*/
static PyObject *
BTreeIter_next(BTreeIter *bi, PyObject *args)
{
PyObject *result = NULL; /* until proven innocent */
BTreeItems *items = bi->pitems;
int i = items->currentoffset;
Bucket *bucket = items->currentbucket;
if (bucket == NULL) /* iteration termination is sticky */
return NULL;
PER_USE_OR_RETURN(bucket, NULL);
if (i >= bucket->len) {
/* We never leave this routine normally with i >= len: somebody
* else mutated the current bucket.
*/
PyErr_SetString(PyExc_RuntimeError,
"the bucket being iterated changed size");
/* Arrange for that this error is sticky too. */
items->currentoffset = INT_MAX;
goto Done;
}
/* Build the result object, from bucket at offset i. */
result = getBucketEntry(bucket, i, items->kind);
/* Advance position for next call. */
if (bucket == items->lastbucket && i >= items->last) {
/* Next call should terminate the iteration. */
Py_DECREF(items->currentbucket);
items->currentbucket = NULL;
}
else {
++i;
if (i >= bucket->len) {
Py_XINCREF(bucket->next);
items->currentbucket = bucket->next;
Py_DECREF(bucket);
i = 0;
}
items->currentoffset = i;
}
Done:
PER_UNUSE(bucket);
return result;
}
static PyObject *
BTreeIter_getiter(PyObject *it)
{
Py_INCREF(it);
return it;
}
static PyTypeObject BTreeIter_Type = {
PyObject_HEAD_INIT(NULL)
0, /* ob_size */
MOD_NAME_PREFIX "-iterator", /* tp_name */
sizeof(BTreeIter), /* tp_basicsize */
0, /* tp_itemsize */
/* methods */
(destructor)BTreeIter_dealloc, /* tp_dealloc */
0, /* tp_print */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_compare */
0, /* tp_repr */
0, /* tp_as_number */
0, /* tp_as_sequence */
0, /* tp_as_mapping */
0, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
0, /*PyObject_GenericGetAttr,*/ /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT, /* tp_flags */
0, /* tp_doc */
0, /* tp_traverse */
0, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
(getiterfunc)BTreeIter_getiter, /* tp_iter */
(iternextfunc)BTreeIter_next, /* tp_iternext */
0, /* tp_methods */
0, /* tp_members */
0, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
};
BTrees/BTreeModuleTemplate.c 0 → 100644
View file @ 703fc7e7
/*****************************************************************************
Copyright (c) 2001, 2002 Zope Foundation and Contributors.
All Rights Reserved.
This software is subject to the provisions of the Zope Public License,
Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
****************************************************************************/
#include "Python.h"
/* include structmember.h for offsetof */
#include "structmember.h"
#ifdef PERSISTENT
#include "persistent/cPersistence.h"
#else
#define PER_USE_OR_RETURN(self, NULL)
#define PER_ALLOW_DEACTIVATION(self)
#define PER_PREVENT_DEACTIVATION(self)
#define PER_DEL(self)
#define PER_USE(O) 1
#define PER_ACCESSED(O) 1
#endif
#include "py24compat.h"
/* So sue me. This pair gets used all over the place, so much so that it
* interferes with understanding non-persistence parts of algorithms.
* PER_UNUSE can be used after a successul PER_USE or PER_USE_OR_RETURN.
* It allows the object to become ghostified, and tells the persistence
* machinery that the object's fields were used recently.
*/
#define PER_UNUSE(OBJ) do { \
PER_ALLOW_DEACTIVATION(OBJ); \
PER_ACCESSED(OBJ); \
} while (0)
/* The tp_name slots of the various BTree types contain the fully
* qualified names of the types, e.g. zodb.btrees.OOBTree.OOBTree.
* The full name is usd to support pickling and because it is not
* possible to modify the __module__ slot of a type dynamically. (This
* may be a bug in Python 2.2).
*
* The MODULE_NAME here used to be "BTrees._". We actually want the module
* name to point to the Python module rather than the C, so the underline
* is now removed.
*/
#define MODULE_NAME "BTrees." MOD_NAME_PREFIX "BTree."
static PyObject *sort_str, *reverse_str, *__setstate___str,
*_bucket_type_str;
static PyObject *ConflictError = NULL;
static void PyVar_Assign(PyObject **v, PyObject *e) { Py_XDECREF(*v); *v=e;}
#define ASSIGN(V,E) PyVar_Assign(&(V),(E))
#define UNLESS(E) if (!(E))
#define OBJECT(O) ((PyObject*)(O))
#define MIN_BUCKET_ALLOC 16
#define MAX_BTREE_SIZE(B) DEFAULT_MAX_BTREE_SIZE
#define MAX_BUCKET_SIZE(B) DEFAULT_MAX_BUCKET_SIZE
#define SameType_Check(O1, O2) ((O1)->ob_type==(O2)->ob_type)
#define ASSERT(C, S, R) if (! (C)) { \
PyErr_SetString(PyExc_AssertionError, (S)); return (R); }
#ifdef NEED_LONG_LONG_SUPPORT
/* Helper code used to support long long instead of int. */
#ifndef PY_LONG_LONG
#error "PY_LONG_LONG required but not defined"
#endif
static int
longlong_check(PyObject *ob)
{
if (PyInt_Check(ob))
return 1;
if (PyLong_Check(ob)) {
/* check magnitude */
PY_LONG_LONG val = PyLong_AsLongLong(ob);
if (val == -1 && PyErr_Occurred())
return 0;
return 1;
}
return 0;
}
static PyObject *
longlong_as_object(PY_LONG_LONG val)
{
static PY_LONG_LONG maxint = 0;
if (maxint == 0)
maxint = PyInt_GetMax();
if ((val > maxint) || (val < (-maxint-1)))
return PyLong_FromLongLong(val);
return PyInt_FromLong((long)val);
}
#endif
/* Various kinds of BTree and Bucket structs are instances of
* "sized containers", and have a common initial layout:
* The stuff needed for all Python objects, or all Persistent objects.
* int size: The maximum number of things that could be contained
* without growing the container.
* int len: The number of things currently contained.
*
* Invariant: 0 <= len <= size.
*
* A sized container typically goes on to declare one or more pointers
* to contiguous arrays with 'size' elements each, the initial 'len' of
* which are currently in use.
*/
#ifdef PERSISTENT
#define sizedcontainer_HEAD \
cPersistent_HEAD \
int size; \
int len;
#else
#define sizedcontainer_HEAD \
PyObject_HEAD \
int size; \
int len;
#endif
/* Nothing is actually of type Sized, but (pointers to) BTree nodes and
* Buckets can be cast to Sized* in contexts that only need to examine
* the members common to all sized containers.
*/
typedef struct Sized_s {
sizedcontainer_HEAD
} Sized;
#define SIZED(O) ((Sized*)(O))
/* A Bucket wraps contiguous vectors of keys and values. Keys are unique,
* and stored in sorted order. The 'values' pointer may be NULL if the
* Bucket is used to implement a set. Buckets serving as leafs of BTrees
* are chained together via 'next', so that the entire BTree contents
* can be traversed in sorted order quickly and easily.
*/
typedef struct Bucket_s {
sizedcontainer_HEAD
struct Bucket_s *next; /* the bucket with the next-larger keys */
KEY_TYPE *keys; /* 'len' keys, in increasing order */
VALUE_TYPE *values; /* 'len' corresponding values; NULL if a set */
} Bucket;
#define BUCKET(O) ((Bucket*)(O))
/* A BTree is complicated. See Maintainer.txt.
*/
typedef struct BTreeItem_s {
KEY_TYPE key;
Sized *child; /* points to another BTree, or to a Bucket of some sort */
} BTreeItem;
typedef struct BTree_s {
sizedcontainer_HEAD
/* firstbucket points to the bucket containing the smallest key in
* the BTree. This is found by traversing leftmost child pointers
* (data[0].child) until reaching a Bucket.
*/
Bucket *firstbucket;
/* The BTree points to 'len' children, via the "child" fields of the data
* array. There are len-1 keys in the 'key' fields, stored in increasing
* order. data[0].key is unused. For i in 0 .. len-1, all keys reachable
* from data[i].child are >= data[i].key and < data[i+1].key, at the
* endpoints pretending that data[0].key is minus infinity and
* data[len].key is positive infinity.
*/
BTreeItem *data;
} BTree;
static PyTypeObject BTreeType;
static PyTypeObject BucketType;
#define BTREE(O) ((BTree*)(O))
/* Use BTREE_SEARCH to find which child pointer to follow.
* RESULT An int lvalue to hold the index i such that SELF->data[i].child
* is the correct node to search next.
* SELF A pointer to a BTree node.
* KEY The key you're looking for, of type KEY_TYPE.
* ONERROR What to do if key comparison raises an exception; for example,
* perhaps 'return NULL'.
*
* See Maintainer.txt for discussion: this is optimized in subtle ways.
* It's recommended that you call this at the start of a routine, waiting
* to check for self->len == 0 after.
*/
#define BTREE_SEARCH(RESULT, SELF, KEY, ONERROR) { \
int _lo = 0; \
int _hi = (SELF)->len; \
int _i, _cmp; \
for (_i = _hi >> 1; _i > _lo; _i = (_lo + _hi) >> 1) { \
TEST_KEY_SET_OR(_cmp, (SELF)->data[_i].key, (KEY)) \
ONERROR; \
if (_cmp < 0) _lo = _i; \
else if (_cmp > 0) _hi = _i; \
else /* equal */ break; \
} \
(RESULT) = _i; \
}
/* SetIteration structs are used in the internal set iteration protocol.
* When you want to iterate over a set or bucket or BTree (even an
* individual key!),
* 1. Declare a new iterator:
* SetIteration si = {0,0,0};
* Using "{0,0,0}" or "{0,0}" appear most common. Only one {0} is
* necssary. At least one must be given so that finiSetIteration() works
* correctly even if you don't get around to calling initSetIteration().
* 2. Initialize it via
* initSetIteration(&si, PyObject *s, useValues)
* It's an error if that returns an int < 0. In case of error on the
* init call, calling finiSetIteration(&si) is optional. But if the
* init call succeeds, you must eventually call finiSetIteration(),
* and whether or not subsequent calls to si.next() fail.
* 3. Get the first element:
* if (si.next(&si) < 0) { there was an error }
* If the set isn't empty, this sets si.position to an int >= 0,
* si.key to the element's key (of type KEY_TYPE), and maybe si.value to
* the element's value (of type VALUE_TYPE). si.value is defined
* iff si.usesValue is true.
* 4. Process all the elements:
* while (si.position >= 0) {
* do something with si.key and/or si.value;
* if (si.next(&si) < 0) { there was an error; }
* }
* 5. Finalize the SetIterator:
* finiSetIteration(&si);
* This is mandatory! si may contain references to iterator objects,
* keys and values, and they must be cleaned up else they'll leak. If
* this were C++ we'd hide that in the destructor, but in C you have to
* do it by hand.
*/
typedef struct SetIteration_s
{
PyObject *set; /* the set, bucket, BTree, ..., being iterated */
int position; /* initialized to 0; set to -1 by next() when done */
int usesValue; /* true iff 'set' has values & we iterate them */
KEY_TYPE key; /* next() sets to next key */
VALUE_TYPE value; /* next() may set to next value */
int (*next)(struct SetIteration_s*); /* function to get next key+value */
} SetIteration;
/* Finish the set iteration protocol. This MUST be called by everyone
* who starts a set iteration, unless the initial call to initSetIteration
* failed; in that case, and only that case, calling finiSetIteration is
* optional.
*/
static void
finiSetIteration(SetIteration *i)
{
assert(i != NULL);
if (i->set == NULL)
return;
Py_DECREF(i->set);
i->set = NULL; /* so it doesn't hurt to call this again */
if (i->position > 0) {
/* next() was called at least once, but didn't finish iterating
* (else position would be negative). So the cached key and
* value need to be cleaned up.
*/
DECREF_KEY(i->key);
if (i->usesValue) {
DECREF_VALUE(i->value);
}
}
i->position = -1; /* stop any stray next calls from doing harm */
}
static PyObject *
IndexError(int i)
{
PyObject *v;
v = PyInt_FromLong(i);
if (!v) {
v = Py_None;
Py_INCREF(v);
}
PyErr_SetObject(PyExc_IndexError, v);
Py_DECREF(v);
return NULL;
}
/* Search for the bucket immediately preceding *current, in the bucket chain
* starting at first. current, *current and first must not be NULL.
*
* Return:
* 1 *current holds the correct bucket; this is a borrowed reference
* 0 no such bucket exists; *current unaltered
* -1 error; *current unaltered
*/
static int
PreviousBucket(Bucket **current, Bucket *first)
{
Bucket *trailing = NULL; /* first travels; trailing follows it */
int result = 0;
assert(current && *current && first);
if (first == *current)
return 0;
do {
trailing = first;
PER_USE_OR_RETURN(first, -1);
first = first->next;
((trailing)->state==cPersistent_STICKY_STATE
&&
((trailing)->state=cPersistent_UPTODATE_STATE));
PER_ACCESSED(trailing);
if (first == *current) {
*current = trailing;
result = 1;
break;
}
} while (first);
return result;
}
static void *
BTree_Malloc(size_t sz)
{
void *r;
ASSERT(sz > 0, "non-positive size malloc", NULL);
r = malloc(sz);
if (r)
return r;
PyErr_NoMemory();
return NULL;
}
static void *
BTree_Realloc(void *p, size_t sz)
{
void *r;
ASSERT(sz > 0, "non-positive size realloc", NULL);
if (p)
r = realloc(p, sz);
else
r = malloc(sz);
UNLESS (r)
PyErr_NoMemory();
return r;
}
/* Shared keyword-argument list for BTree/Bucket
* (iter)?(keys|values|items)
*/
static char *search_keywords[] = {"min", "max",
"excludemin", "excludemax",
0};
#include "BTreeItemsTemplate.c"
#include "BucketTemplate.c"
#include "SetTemplate.c"
#include "BTreeTemplate.c"
#include "TreeSetTemplate.c"
#include "SetOpTemplate.c"
#include "MergeTemplate.c"
static struct PyMethodDef module_methods[] = {
{"difference", (PyCFunction) difference_m, METH_VARARGS,
"difference(o1, o2) -- "
"compute the difference between o1 and o2"
},
{"union", (PyCFunction) union_m, METH_VARARGS,
"union(o1, o2) -- compute the union of o1 and o2\n"
},
{"intersection", (PyCFunction) intersection_m, METH_VARARGS,
"intersection(o1, o2) -- "
"compute the intersection of o1 and o2"
},
#ifdef MERGE
{"weightedUnion", (PyCFunction) wunion_m, METH_VARARGS,
"weightedUnion(o1, o2 [, w1, w2]) -- compute the union of o1 and o2\n"
"\nw1 and w2 are weights."
},
{"weightedIntersection", (PyCFunction) wintersection_m, METH_VARARGS,
"weightedIntersection(o1, o2 [, w1, w2]) -- "
"compute the intersection of o1 and o2\n"
"\nw1 and w2 are weights."
},
#endif
#ifdef MULTI_INT_UNION
{"multiunion", (PyCFunction) multiunion_m, METH_VARARGS,
"multiunion(seq) -- compute union of a sequence of integer sets.\n"
"\n"
"Each element of seq must be an integer set, or convertible to one\n"
"via the set iteration protocol. The union returned is an IISet."
},
#endif
{NULL, NULL} /* sentinel */
};
static char BTree_module_documentation[] =
"\n"
MASTER_ID
BTREEITEMSTEMPLATE_C
"$Id$\n"
BTREETEMPLATE_C
BUCKETTEMPLATE_C
KEYMACROS_H
MERGETEMPLATE_C
SETOPTEMPLATE_C
SETTEMPLATE_C
TREESETTEMPLATE_C
VALUEMACROS_H
BTREEITEMSTEMPLATE_C
;
int
init_persist_type(PyTypeObject *type)
{
type->ob_type = &PyType_Type;
type->tp_base = cPersistenceCAPI->pertype;
if (PyType_Ready(type) < 0)
return 0;
return 1;
}
void
INITMODULE (void)
{
PyObject *m, *d, *c;
#ifdef KEY_TYPE_IS_PYOBJECT
object_ = PyTuple_GetItem(Py_None->ob_type->tp_bases, 0);
if (object_ == NULL)
return;
#endif
sort_str = PyString_InternFromString("sort");
if (!sort_str)
return;
reverse_str = PyString_InternFromString("reverse");
if (!reverse_str)
return;
__setstate___str = PyString_InternFromString("__setstate__");
if (!__setstate___str)
return;
_bucket_type_str = PyString_InternFromString("_bucket_type");
if (!_bucket_type_str)
return;
/* Grab the ConflictError class */
m = PyImport_ImportModule("ZODB.POSException");
if (m != NULL) {
c = PyObject_GetAttrString(m, "BTreesConflictError");
if (c != NULL)
ConflictError = c;
Py_DECREF(m);
}
if (ConflictError == NULL) {
Py_INCREF(PyExc_ValueError);
ConflictError=PyExc_ValueError;
}
/* Initialize the PyPersist_C_API and the type objects. */
cPersistenceCAPI = PyCObject_Import("persistent.cPersistence", "CAPI");
if (cPersistenceCAPI == NULL)
return;
BTreeItemsType.ob_type = &PyType_Type;
BTreeIter_Type.ob_type = &PyType_Type;
BTreeIter_Type.tp_getattro = PyObject_GenericGetAttr;
BucketType.tp_new = PyType_GenericNew;
SetType.tp_new = PyType_GenericNew;
BTreeType.tp_new = PyType_GenericNew;
TreeSetType.tp_new = PyType_GenericNew;
if (!init_persist_type(&BucketType))
return;
if (!init_persist_type(&BTreeType))
return;
if (!init_persist_type(&SetType))
return;
if (!init_persist_type(&TreeSetType))
return;
if (PyDict_SetItem(BTreeType.tp_dict, _bucket_type_str,
(PyObject *)&BucketType) < 0) {
fprintf(stderr, "btree failed\n");
return;
}
if (PyDict_SetItem(TreeSetType.tp_dict, _bucket_type_str,
(PyObject *)&SetType) < 0) {
fprintf(stderr, "bucket failed\n");
return;
}
/* Create the module and add the functions */
m = Py_InitModule4("_" MOD_NAME_PREFIX "BTree",
module_methods, BTree_module_documentation,
(PyObject *)NULL, PYTHON_API_VERSION);
/* Add some symbolic constants to the module */
d = PyModule_GetDict(m);
if (PyDict_SetItemString(d, MOD_NAME_PREFIX "Bucket",
(PyObject *)&BucketType) < 0)
return;
if (PyDict_SetItemString(d, MOD_NAME_PREFIX "BTree",
(PyObject *)&BTreeType) < 0)
return;
if (PyDict_SetItemString(d, MOD_NAME_PREFIX "Set",
(PyObject *)&SetType) < 0)
return;
if (PyDict_SetItemString(d, MOD_NAME_PREFIX "TreeSet",
(PyObject *)&TreeSetType) < 0)
return;
if (PyDict_SetItemString(d, MOD_NAME_PREFIX "TreeIterator",
(PyObject *)&BTreeIter_Type) < 0)
return;
/* We also want to be able to access these constants without the prefix
* so that code can more easily exchange modules (particularly the integer
* and long modules, but also others). The TreeIterator is only internal,
* so we don't bother to expose that.
*/
if (PyDict_SetItemString(d, "Bucket",
(PyObject *)&BucketType) < 0)
return;
if (PyDict_SetItemString(d, "BTree",
(PyObject *)&BTreeType) < 0)
return;
if (PyDict_SetItemString(d, "Set",
(PyObject *)&SetType) < 0)
return;
if (PyDict_SetItemString(d, "TreeSet",
(PyObject *)&TreeSetType) < 0)
return;
#if defined(ZODB_64BIT_INTS) && defined(NEED_LONG_LONG_SUPPORT)
if (PyDict_SetItemString(d, "using64bits", Py_True) < 0)
return;
#else
if (PyDict_SetItemString(d, "using64bits", Py_False) < 0)
return;
#endif
}
BTrees/BTreeTemplate.c 0 → 100644
View file @ 703fc7e7
/*****************************************************************************
Copyright (c) 2001, 2002 Zope Foundation and Contributors.
All Rights Reserved.
This software is subject to the provisions of the Zope Public License,
Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
****************************************************************************/
#define BTREETEMPLATE_C "$Id$\n"
/* Sanity-check a BTree. This is a private helper for BTree_check. Return:
* -1 Error. If it's an internal inconsistency in the BTree,
* AssertionError is set.
* 0 No problem found.
*
* nextbucket is the bucket "one beyond the end" of the BTree; the last bucket
* directly reachable from following right child pointers *should* be linked
* to nextbucket (and this is checked).
*/
static int
BTree_check_inner(BTree *self, Bucket *nextbucket)
{
int i;
Bucket *bucketafter;
Sized *child;
char *errormsg = "internal error"; /* someone should have overriden */
Sized *activated_child = NULL;
int result = -1; /* until proved innocent */
#define CHECK(CONDITION, ERRORMSG) \
if (!(CONDITION)) { \
errormsg = (ERRORMSG); \
goto Error; \
}
PER_USE_OR_RETURN(self, -1);
CHECK(self->len >= 0, "BTree len < 0");
CHECK(self->len <= self->size, "BTree len > size");
if (self->len == 0) {
/* Empty BTree. */
CHECK(self->firstbucket == NULL,
"Empty BTree has non-NULL firstbucket");
result = 0;
goto Done;
}
/* Non-empty BTree. */
CHECK(self->firstbucket != NULL, "Non-empty BTree has NULL firstbucket");
/* Obscure: The first bucket is pointed to at least by self->firstbucket
* and data[0].child of whichever BTree node it's a child of. However,
* if persistence is enabled then the latter BTree node may be a ghost
* at this point, and so its pointers "don't count": we can only rely
* on self's pointers being intact.
*/
#ifdef PERSISTENT
CHECK(self->firstbucket->ob_refcnt >= 1,
"Non-empty BTree firstbucket has refcount < 1");
#else
CHECK(self->firstbucket->ob_refcnt >= 2,
"Non-empty BTree firstbucket has refcount < 2");
#endif
for (i = 0; i < self->len; ++i) {
CHECK(self->data[i].child != NULL, "BTree has NULL child");
}
if (SameType_Check(self, self->data[0].child)) {
/* Our children are also BTrees. */
child = self->data[0].child;
UNLESS (PER_USE(child)) goto Done;
activated_child = child;
CHECK(self->firstbucket == BTREE(child)->firstbucket,
"BTree has firstbucket different than "
"its first child's firstbucket");
PER_ALLOW_DEACTIVATION(child);
activated_child = NULL;
for (i = 0; i < self->len; ++i) {
child = self->data[i].child;
CHECK(SameType_Check(self, child),
"BTree children have different types");
if (i == self->len - 1)
bucketafter = nextbucket;
else {
BTree *child2 = BTREE(self->data[i+1].child);
UNLESS (PER_USE(child2)) goto Done;
bucketafter = child2->firstbucket;
PER_ALLOW_DEACTIVATION(child2);
}
if (BTree_check_inner(BTREE(child), bucketafter) < 0) goto Done;
}
}
else {
/* Our children are buckets. */
CHECK(self->firstbucket == BUCKET(self->data[0].child),
"Bottom-level BTree node has inconsistent firstbucket belief");
for (i = 0; i < self->len; ++i) {
child = self->data[i].child;
UNLESS (PER_USE(child)) goto Done;
activated_child = child;
CHECK(!SameType_Check(self, child),
"BTree children have different types");
CHECK(child->len >= 1, "Bucket length < 1"); /* no empty buckets! */
CHECK(child->len <= child->size, "Bucket len > size");
#ifdef PERSISTENT
CHECK(child->ob_refcnt >= 1, "Bucket has refcount < 1");
#else
CHECK(child->ob_refcnt >= 2, "Bucket has refcount < 2");
#endif
if (i == self->len - 1)
bucketafter = nextbucket;
else
bucketafter = BUCKET(self->data[i+1].child);
CHECK(BUCKET(child)->next == bucketafter,
"Bucket next pointer is damaged");
PER_ALLOW_DEACTIVATION(child);
activated_child = NULL;
}
}
result = 0;
goto Done;
Error:
PyErr_SetString(PyExc_AssertionError, errormsg);
result = -1;
Done:
/* No point updating access time -- this isn't a "real" use. */
PER_ALLOW_DEACTIVATION(self);
if (activated_child) {
PER_ALLOW_DEACTIVATION(activated_child);
}
return result;
#undef CHECK
}
/* Sanity-check a BTree. This is the ._check() method. Return:
* NULL Error. If it's an internal inconsistency in the BTree,
* AssertionError is set.
* Py_None No problem found.
*/
static PyObject*
BTree_check(BTree *self)
{
PyObject *result = NULL;
int i = BTree_check_inner(self, NULL);
if (i >= 0) {
result = Py_None;
Py_INCREF(result);
}
return result;
}
/*
** _BTree_get
**
** Search a BTree.
**
** Arguments
** self a pointer to a BTree
** keyarg the key to search for, as a Python object
** has_key true/false; when false, try to return the associated
** value; when true, return a boolean
** Return
** When has_key false:
** If key exists, its associated value.
** If key doesn't exist, NULL and KeyError is set.
** When has_key true:
** A Python int is returned in any case.
** If key exists, the depth of the bucket in which it was found.
** If key doesn't exist, 0.
*/
static PyObject *
_BTree_get(BTree *self, PyObject *keyarg, int has_key)
{
KEY_TYPE key;
PyObject *result = NULL; /* guilty until proved innocent */
int copied = 1;
COPY_KEY_FROM_ARG(key, keyarg, copied);
UNLESS (copied) return NULL;
PER_USE_OR_RETURN(self, NULL);
if (self->len == 0) {
/* empty BTree */
if (has_key)
result = PyInt_FromLong(0);
else
PyErr_SetObject(PyExc_KeyError, keyarg);
}
else {
for (;;) {
int i;
Sized *child;
BTREE_SEARCH(i, self, key, goto Done);
child = self->data[i].child;
has_key += has_key != 0; /* bump depth counter, maybe */
if (SameType_Check(self, child)) {
PER_UNUSE(self);
self = BTREE(child);
PER_USE_OR_RETURN(self, NULL);
}
else {
result = _bucket_get(BUCKET(child), keyarg, has_key);
break;
}
}
}
Done:
PER_UNUSE(self);
return result;
}
static PyObject *
BTree_get(BTree *self, PyObject *key)
{
return _BTree_get(self, key, 0);
}
/* Create a new bucket for the BTree or TreeSet using the class attribute
_bucket_type, which is normally initialized to BucketType or SetType
as appropriate.
*/
static Sized *
BTree_newBucket(BTree *self)
{
PyObject *factory;
Sized *result;
/* _bucket_type_str defined in BTreeModuleTemplate.c */
factory = PyObject_GetAttr((PyObject *)self->ob_type, _bucket_type_str);
if (factory == NULL)
return NULL;
/* TODO: Should we check that the factory actually returns something
of the appropriate type? How? The C code here is going to
depend on any custom bucket type having the same layout at the
C level.
*/
result = SIZED(PyObject_CallObject(factory, NULL));
Py_DECREF(factory);
return result;
}
/*
* Move data from the current BTree, from index onward, to the newly created
* BTree 'next'. self and next must both be activated. If index is OOB (< 0
* or >= self->len), use self->len / 2 as the index (i.e., split at the
* midpoint). self must have at least 2 children on entry, and index must
* be such that self and next each have at least one child at exit. self's
* accessed time is updated.
*
* Return:
* -1 error
* 0 OK
*/
static int
BTree_split(BTree *self, int index, BTree *next)
{
int next_size;
Sized *child;
if (index < 0 || index >= self->len)
index = self->len / 2;
next_size = self->len - index;
ASSERT(index > 0, "split creates empty tree", -1);
ASSERT(next_size > 0, "split creates empty tree", -1);
next->data = BTree_Malloc(sizeof(BTreeItem) * next_size);
if (!next->data)
return -1;
memcpy(next->data, self->data + index, sizeof(BTreeItem) * next_size);
next->size = next_size; /* but don't set len until we succeed */
/* Set next's firstbucket. self->firstbucket is still correct. */
child = next->data[0].child;
if (SameType_Check(self, child)) {
PER_USE_OR_RETURN(child, -1);
next->firstbucket = BTREE(child)->firstbucket;
PER_UNUSE(child);
}
else
next->firstbucket = BUCKET(child);
Py_INCREF(next->firstbucket);
next->len = next_size;
self->len = index;
return PER_CHANGED(self) >= 0 ? 0 : -1;
}
/* Fwd decl -- BTree_grow and BTree_split_root reference each other. */
static int BTree_grow(BTree *self, int index, int noval);
/* Split the root. This is a little special because the root isn't a child
* of anything else, and the root needs to retain its object identity. So
* this routine moves the root's data into a new child, and splits the
* latter. This leaves the root with two children.
*
* Return:
* 0 OK
* -1 error
*
* CAUTION: The caller must call PER_CHANGED on self.
*/
static int
BTree_split_root(BTree *self, int noval)
{
BTree *child;
BTreeItem *d;
/* Create a child BTree, and a new data vector for self. */
child = BTREE(PyObject_CallObject(OBJECT(self->ob_type), NULL));
if (!child) return -1;
d = BTree_Malloc(sizeof(BTreeItem) * 2);
if (!d) {
Py_DECREF(child);
return -1;
}
/* Move our data to new BTree. */
child->size = self->size;
child->len = self->len;
child->data = self->data;
child->firstbucket = self->firstbucket;
Py_INCREF(child->firstbucket);
/* Point self to child and split the child. */
self->data = d;
self->len = 1;
self->size = 2;
self->data[0].child = SIZED(child); /* transfers reference ownership */
return BTree_grow(self, 0, noval);
}
/*
** BTree_grow
**
** Grow a BTree
**
** Arguments: self The BTree
** index self->data[index].child needs to be split. index
** must be 0 if self is empty (len == 0), and a new
** empty bucket is created then.
** noval Boolean; is this a set (true) or mapping (false)?
**
** Returns: 0 on success
** -1 on failure
**
** CAUTION: If self is empty on entry, this routine adds an empty bucket.
** That isn't a legitimate BTree; if the caller doesn't put something in
** in the bucket (say, because of a later error), the BTree must be cleared
** to get rid of the empty bucket.
*/
static int
BTree_grow(BTree *self, int index, int noval)
{
int i;
Sized *v, *e = 0;
BTreeItem *d;
if (self->len == self->size) {
if (self->size) {
d = BTree_Realloc(self->data, sizeof(BTreeItem) * self->size * 2);
if (d == NULL)
return -1;
self->data = d;
self->size *= 2;
}
else {
d = BTree_Malloc(sizeof(BTreeItem) * 2);
if (d == NULL)
return -1;
self->data = d;
self->size = 2;
}
}
if (self->len) {
d = self->data + index;
v = d->child;
/* Create a new object of the same type as the target value */
e = (Sized *)PyObject_CallObject((PyObject *)v->ob_type, NULL);
if (e == NULL)
return -1;
UNLESS(PER_USE(v)) {
Py_DECREF(e);
return -1;
}
/* Now split between the original (v) and the new (e) at the midpoint*/
if (SameType_Check(self, v))
i = BTree_split((BTree *)v, -1, (BTree *)e);
else
i = bucket_split((Bucket *)v, -1, (Bucket *)e);
PER_ALLOW_DEACTIVATION(v);
if (i < 0) {
Py_DECREF(e);
assert(PyErr_Occurred());
return -1;
}
index++;
d++;
if (self->len > index) /* Shift up the old values one array slot */
memmove(d+1, d, sizeof(BTreeItem)*(self->len-index));
if (SameType_Check(self, v)) {
COPY_KEY(d->key, BTREE(e)->data->key);
/* We take the unused reference from e, so there's no
reason to INCREF!
*/
/* INCREF_KEY(self->data[1].key); */
}
else {
COPY_KEY(d->key, BUCKET(e)->keys[0]);
INCREF_KEY(d->key);
}
d->child = e;
self->len++;
if (self->len >= MAX_BTREE_SIZE(self) * 2) /* the root is huge */
return BTree_split_root(self, noval);
}
else {
/* The BTree is empty. Create an empty bucket. See CAUTION in
* the comments preceding.
*/
assert(index == 0);
d = self->data;
d->child = BTree_newBucket(self);
if (d->child == NULL)
return -1;
self->len = 1;
Py_INCREF(d->child);
self->firstbucket = (Bucket *)d->child;
}
return 0;
}
/* Return the rightmost bucket reachable from following child pointers
* from self. The caller gets a new reference to this bucket. Note that
* bucket 'next' pointers are not followed: if self is an interior node
* of a BTree, this returns the rightmost bucket in that node's subtree.
* In case of error, returns NULL.
*
* self must not be a ghost; this isn't checked. The result may be a ghost.
*
* Pragmatics: Note that the rightmost bucket's last key is the largest
* key in self's subtree.
*/
static Bucket *
BTree_lastBucket(BTree *self)
{
Sized *pchild;
Bucket *result;
UNLESS (self->data && self->len) {
IndexError(-1); /* is this the best action to take? */
return NULL;
}
pchild = self->data[self->len - 1].child;
if (SameType_Check(self, pchild)) {
self = BTREE(pchild);
PER_USE_OR_RETURN(self, NULL);
result = BTree_lastBucket(self);
PER_UNUSE(self);
}
else {
Py_INCREF(pchild);
result = BUCKET(pchild);
}
return result;
}
static int
BTree_deleteNextBucket(BTree *self)
{
Bucket *b;
UNLESS (PER_USE(self)) return -1;
b = BTree_lastBucket(self);
if (b == NULL)
goto err;
if (Bucket_deleteNextBucket(b) < 0)
goto err;
Py_DECREF(b);
PER_UNUSE(self);
return 0;
err:
Py_XDECREF(b);
PER_ALLOW_DEACTIVATION(self);
return -1;
}
/*
** _BTree_clear
**
** Clears out all of the values in the BTree (firstbucket, keys, and children);
** leaving self an empty BTree.
**
** Arguments: self The BTree
**
** Returns: 0 on success
** -1 on failure
**
** Internal: Deallocation order is important. The danger is that a long
** list of buckets may get freed "at once" via decref'ing the first bucket,
** in which case a chain of consequenct Py_DECREF calls may blow the stack.
** Luckily, every bucket has a refcount of at least two, one due to being a
** BTree node's child, and another either because it's not the first bucket in
** the chain (so the preceding bucket points to it), or because firstbucket
** points to it. By clearing in the natural depth-first, left-to-right
** order, the BTree->bucket child pointers prevent Py_DECREF(bucket->next)
** calls from freeing bucket->next, and the maximum stack depth is equal
** to the height of the tree.
**/
static int
_BTree_clear(BTree *self)
{
const int len = self->len;
if (self->firstbucket) {
/* Obscure: The first bucket is pointed to at least by
* self->firstbucket and data[0].child of whichever BTree node it's
* a child of. However, if persistence is enabled then the latter
* BTree node may be a ghost at this point, and so its pointers "don't
* count": we can only rely on self's pointers being intact.
*/
#ifdef PERSISTENT
ASSERT(self->firstbucket->ob_refcnt > 0,
"Invalid firstbucket pointer", -1);
#else
ASSERT(self->firstbucket->ob_refcnt > 1,
"Invalid firstbucket pointer", -1);
#endif
Py_DECREF(self->firstbucket);
self->firstbucket = NULL;
}
if (self->data) {
int i;
if (len > 0) { /* 0 is special because key 0 is trash */
Py_DECREF(self->data[0].child);
}
for (i = 1; i < len; i++) {
#ifdef KEY_TYPE_IS_PYOBJECT
DECREF_KEY(self->data[i].key);
#endif
Py_DECREF(self->data[i].child);
}
free(self->data);
self->data = NULL;
}
self->len = self->size = 0;
return 0;
}
/*
Set (value != 0) or delete (value=0) a tree item.
If unique is non-zero, then only change if the key is
new.
If noval is non-zero, then don't set a value (the tree
is a set).
Return:
-1 error
0 successful, and number of entries didn't change
>0 successful, and number of entries did change
Internal
There are two distinct return values > 0:
1 Successful, number of entries changed, but firstbucket did not go away.
2 Successful, number of entries changed, firstbucket did go away.
This can only happen on a delete (value == NULL). The caller may
need to change its own firstbucket pointer, and in any case *someone*
needs to adjust the 'next' pointer of the bucket immediately preceding
the bucket that went away (it needs to point to the bucket immediately
following the bucket that went away).
*/
static int
_BTree_set(BTree *self, PyObject *keyarg, PyObject *value,
int unique, int noval)
{
int changed = 0; /* did I mutate? */
int min; /* index of child I searched */
BTreeItem *d; /* self->data[min] */
int childlength; /* len(self->data[min].child) */
int status; /* our return value; and return value from callee */
int self_was_empty; /* was self empty at entry? */
KEY_TYPE key;
int copied = 1;
COPY_KEY_FROM_ARG(key, keyarg, copied);
if (!copied) return -1;
PER_USE_OR_RETURN(self, -1);
self_was_empty = self->len == 0;
if (self_was_empty) {
/* We're empty. Make room. */
if (value) {
if (BTree_grow(self, 0, noval) < 0)
goto Error;
}
else {
/* Can't delete a key from an empty BTree. */
PyErr_SetObject(PyExc_KeyError, keyarg);
goto Error;
}
}
/* Find the right child to search, and hand the work off to it. */
BTREE_SEARCH(min, self, key, goto Error);
d = self->data + min;
#ifdef PERSISTENT
PER_READCURRENT(self, goto Error);
#endif
if (SameType_Check(self, d->child))
status = _BTree_set(BTREE(d->child), keyarg, value, unique, noval);
else {
int bucket_changed = 0;
status = _bucket_set(BUCKET(d->child), keyarg,
value, unique, noval, &bucket_changed);
#ifdef PERSISTENT
/* If a BTree contains only a single bucket, BTree.__getstate__()
* includes the bucket's entire state, and the bucket doesn't get
* an oid of its own. So if we have a single oid-less bucket that
* changed, it's *our* oid that should be marked as changed -- the
* bucket doesn't have one.
*/
if (bucket_changed
&& self->len == 1
&& self->data[0].child->oid == NULL)
{
changed = 1;
}
#endif
}
if (status == 0) goto Done;
if (status < 0) goto Error;
assert(status == 1 || status == 2);
/* The child changed size. Get its new size. Note that since the tree
* rooted at the child changed size, so did the tree rooted at self:
* our status must be >= 1 too.
*/
UNLESS(PER_USE(d->child)) goto Error;
childlength = d->child->len;
PER_UNUSE(d->child);
if (value) {
/* A bucket got bigger -- if it's "too big", split it. */
int toobig;
assert(status == 1); /* can be 2 only on deletes */
if (SameType_Check(self, d->child))
toobig = childlength > MAX_BTREE_SIZE(d->child);
else
toobig = childlength > MAX_BUCKET_SIZE(d->child);
if (toobig) {
if (BTree_grow(self, min, noval) < 0) goto Error;
changed = 1; /* BTree_grow mutated self */
}
goto Done; /* and status still == 1 */
}
/* A bucket got smaller. This is much harder, and despite that we
* don't try to rebalance the tree.
*/
if (min && childlength)
{ /* We removed a key. but the node child is non-empty. If the
deleted key is the node key, then update the node key using
the smallest key of the node child.
This doesn't apply to the 0th node, whos key is unused.
*/
int _cmp = 1;
TEST_KEY_SET_OR(_cmp, key, d->key) goto Error;
if (_cmp == 0)
{ /* Need to replace key with first key from child */
Bucket *bucket;
if (SameType_Check(self, d->child))
{
UNLESS(PER_USE(d->child)) goto Error;
bucket = BTREE(d->child)->firstbucket;
PER_UNUSE(d->child);
}
else
bucket = BUCKET(d->child);
UNLESS(PER_USE(bucket)) goto Error;
DECREF_KEY(d->key);
COPY_KEY(d->key, bucket->keys[0]);
INCREF_KEY(d->key);
PER_UNUSE(bucket);
if (PER_CHANGED(self) < 0) goto Error;
}
}
if (status == 2) {
/* The child must be a BTree because bucket.set never returns 2 */
/* Two problems to solve: May have to adjust our own firstbucket,
* and the bucket that went away needs to get unlinked.
*/
if (min) {
/* This wasn't our firstbucket, so no need to adjust ours (note
* that it can't be the firstbucket of any node above us either).
* Tell "the tree to the left" to do the unlinking.
*/
if (BTree_deleteNextBucket(BTREE(d[-1].child)) < 0) goto Error;
status = 1; /* we solved the child's firstbucket problem */
}
else {
/* This was our firstbucket. Update to new firstbucket value. */
Bucket *nextbucket;
UNLESS(PER_USE(d->child)) goto Error;
nextbucket = BTREE(d->child)->firstbucket;
PER_UNUSE(d->child);
Py_XINCREF(nextbucket);
Py_DECREF(self->firstbucket);
self->firstbucket = nextbucket;
changed = 1;
/* The caller has to do the unlinking -- we can't. Also, since
* it was our firstbucket, it may also be theirs.
*/
assert(status == 2);
}
}
/* If the child isn't empty, we're done! We did all that was possible for
* us to do with the firstbucket problems the child gave us, and since the
* child isn't empty don't create any new firstbucket problems of our own.
*/
if (childlength) goto Done;
/* The child became empty: we need to remove it from self->data.
* But first, if we're a bottom-level node, we've got more bucket-fiddling
* to set up.
*/
if (! SameType_Check(self, d->child)) {
/* We're about to delete a bucket, so need to adjust bucket pointers. */
if (min) {
/* It's not our first bucket, so we can tell the previous
* bucket to adjust its reference to it. It can't be anyone
* else's first bucket either, so the caller needn't do anything.
*/
if (Bucket_deleteNextBucket(BUCKET(d[-1].child)) < 0) goto Error;
/* status should be 1, and already is: if it were 2, the
* block above would have set it to 1 in its min != 0 branch.
*/
assert(status == 1);
}
else {
Bucket *nextbucket;
/* It's our first bucket. We can't unlink it directly. */
/* 'changed' will be set true by the deletion code following. */
UNLESS(PER_USE(d->child)) goto Error;
nextbucket = BUCKET(d->child)->next;
PER_UNUSE(d->child);
Py_XINCREF(nextbucket);
Py_DECREF(self->firstbucket);
self->firstbucket = nextbucket;
status = 2; /* we're giving our caller a new firstbucket problem */
}
}
/* Remove the child from self->data. */
Py_DECREF(d->child);
#ifdef KEY_TYPE_IS_PYOBJECT
if (min) {
DECREF_KEY(d->key);
}
else if (self->len > 1) {
/* We're deleting the first child of a BTree with more than one
* child. The key at d+1 is about to be shifted into slot 0,
* and hence never to be referenced again (the key in slot 0 is
* trash).
*/
DECREF_KEY((d+1)->key);
}
/* Else min==0 and len==1: we're emptying the BTree entirely, and
* there is no key in need of decrefing.
*/
#endif
--self->len;
if (min < self->len)
memmove(d, d+1, (self->len - min) * sizeof(BTreeItem));
changed = 1;
Done:
#ifdef PERSISTENT
if (changed) {
if (PER_CHANGED(self) < 0) goto Error;
}
#endif
PER_UNUSE(self);
return status;
Error:
assert(PyErr_Occurred());
if (self_was_empty) {
/* BTree_grow may have left the BTree in an invalid state. Make
* sure the tree is a legitimate empty tree.
*/
_BTree_clear(self);
}
PER_UNUSE(self);
return -1;
}
/*
** BTree_setitem
**
** wrapper for _BTree_set
**
** Arguments: self The BTree
** key The key to insert
** v The value to insert
**
** Returns -1 on failure
** 0 on success
*/
static int
BTree_setitem(BTree *self, PyObject *key, PyObject *v)
{
if (_BTree_set(self, key, v, 0, 0) < 0)
return -1;
return 0;
}
#ifdef PERSISTENT
static PyObject *
BTree__p_deactivate(BTree *self, PyObject *args, PyObject *keywords)
{
int ghostify = 1;
PyObject *force = NULL;
if (args && PyTuple_GET_SIZE(args) > 0) {
PyErr_SetString(PyExc_TypeError,
"_p_deactivate takes not positional arguments");
return NULL;
}
if (keywords) {
int size = PyDict_Size(keywords);
force = PyDict_GetItemString(keywords, "force");
if (force)
size--;
if (size) {
PyErr_SetString(PyExc_TypeError,
"_p_deactivate only accepts keyword arg force");
return NULL;
}
}
if (self->jar && self->oid) {
ghostify = self->state == cPersistent_UPTODATE_STATE;
if (!ghostify && force) {
if (PyObject_IsTrue(force))
ghostify = 1;
if (PyErr_Occurred())
return NULL;
}
if (ghostify) {
if (_BTree_clear(self) < 0)
return NULL;
PER_GHOSTIFY(self);
}
}
Py_INCREF(Py_None);
return Py_None;
}
#endif
static PyObject *
BTree_clear(BTree *self)
{
UNLESS (PER_USE(self)) return NULL;
if (self->len)
{
if (_BTree_clear(self) < 0)
goto err;
if (PER_CHANGED(self) < 0)
goto err;
}
PER_UNUSE(self);
Py_INCREF(Py_None);
return Py_None;
err:
PER_UNUSE(self);
return NULL;
}
/*
* Return:
*
* For an empty BTree (self->len == 0), None.
*
* For a BTree with one child (self->len == 1), and that child is a bucket,
* and that bucket has a NULL oid, a one-tuple containing a one-tuple
* containing the bucket's state:
*
* (
* (
* child[0].__getstate__(),
* ),
* )
*
* Else a two-tuple. The first element is a tuple interleaving the BTree's
* keys and direct children, of size 2*self->len - 1 (key[0] is unused and
* is not saved). The second element is the firstbucket:
*
* (
* (child[0], key[1], child[1], key[2], child[2], ...,
* key[len-1], child[len-1]),
* self->firstbucket
* )
*
* In the above, key[i] means self->data[i].key, and similarly for child[i].
*/
static PyObject *
BTree_getstate(BTree *self)
{
PyObject *r = NULL;
PyObject *o;
int i, l;
UNLESS (PER_USE(self)) return NULL;
if (self->len) {
r = PyTuple_New(self->len * 2 - 1);
if (r == NULL)
goto err;
if (self->len == 1
&& self->data->child->ob_type != self->ob_type
#ifdef PERSISTENT
&& BUCKET(self->data->child)->oid == NULL
#endif
) {
/* We have just one bucket. Save its data directly. */
o = bucket_getstate((Bucket *)self->data->child);
if (o == NULL)
goto err;
PyTuple_SET_ITEM(r, 0, o);
ASSIGN(r, Py_BuildValue("(O)", r));
}
else {
for (i=0, l=0; i < self->len; i++) {
if (i) {
COPY_KEY_TO_OBJECT(o, self->data[i].key);
PyTuple_SET_ITEM(r, l, o);
l++;
}
o = (PyObject *)self->data[i].child;
Py_INCREF(o);
PyTuple_SET_ITEM(r,l,o);
l++;
}
ASSIGN(r, Py_BuildValue("OO", r, self->firstbucket));
}
}
else {
r = Py_None;
Py_INCREF(r);
}
PER_UNUSE(self);
return r;
err:
PER_UNUSE(self);
Py_XDECREF(r);
return NULL;
}
static int
_BTree_setstate(BTree *self, PyObject *state, int noval)
{
PyObject *items, *firstbucket = NULL;
BTreeItem *d;
int len, l, i, copied=1;
if (_BTree_clear(self) < 0)
return -1;
/* The state of a BTree can be one of the following:
None -- an empty BTree
A one-tuple -- a single bucket btree
A two-tuple -- a BTree with more than one bucket
See comments for BTree_getstate() for the details.
*/
if (state == Py_None)
return 0;
if (!PyArg_ParseTuple(state, "O|O:__setstate__", &items, &firstbucket))
return -1;
if (!PyTuple_Check(items)) {
PyErr_SetString(PyExc_TypeError,
"tuple required for first state element");
return -1;
}
len = PyTuple_Size(items);
if (len < 0)
return -1;
len = (len + 1) / 2;
assert(len > 0); /* If the BTree is empty, it's state is None. */
assert(self->size == 0); /* We called _BTree_clear(). */
self->data = BTree_Malloc(sizeof(BTreeItem) * len);
if (self->data == NULL)
return -1;
self->size = len;
for (i = 0, d = self->data, l = 0; i < len; i++, d++) {
PyObject *v;
if (i) { /* skip the first key slot */
COPY_KEY_FROM_ARG(d->key, PyTuple_GET_ITEM(items, l), copied);
l++;
if (!copied)
return -1;
INCREF_KEY(d->key);
}
v = PyTuple_GET_ITEM(items, l);
if (PyTuple_Check(v)) {
/* Handle the special case in __getstate__() for a BTree
with a single bucket. */
d->child = BTree_newBucket(self);
if (!d->child)
return -1;
if (noval) {
if (_set_setstate(BUCKET(d->child), v) < 0)
return -1;
}
else {
if (_bucket_setstate(BUCKET(d->child), v) < 0)
return -1;
}
}
else {
d->child = (Sized *)v;
Py_INCREF(v);
}
l++;
}
if (!firstbucket)
firstbucket = (PyObject *)self->data->child;
if (!PyObject_IsInstance(firstbucket, (PyObject *)
(noval ? &SetType : &BucketType))) {
PyErr_SetString(PyExc_TypeError,
"No firstbucket in non-empty BTree");
return -1;
}
self->firstbucket = BUCKET(firstbucket);
Py_INCREF(firstbucket);
#ifndef PERSISTENT
/* firstbucket is also the child of some BTree node, but that node may
* be a ghost if persistence is enabled.
*/
assert(self->firstbucket->ob_refcnt > 1);
#endif
self->len = len;
return 0;
}
static PyObject *
BTree_setstate(BTree *self, PyObject *arg)
{
int r;
PER_PREVENT_DEACTIVATION(self);
r = _BTree_setstate(self, arg, 0);
PER_UNUSE(self);
if (r < 0)
return NULL;
Py_INCREF(Py_None);
return Py_None;
}
#ifdef PERSISTENT
/* Recognize the special cases of a BTree that's empty or contains a single
* bucket. In the former case, return a borrowed reference to Py_None.
* In this single-bucket case, the bucket state is embedded directly in the
* BTree state, like so:
*
* (
* (
* thebucket.__getstate__(),
* ),
* )
*
* When this obtains, return a borrowed reference to thebucket.__getstate__().
* Else return NULL with an exception set. The exception should always be
* ConflictError then, but may be TypeError if the state makes no sense at all
* for a BTree (corrupted or hostile state).
*/
PyObject *
get_bucket_state(PyObject *t)
{
if (t == Py_None)
return Py_None; /* an empty BTree */
if (! PyTuple_Check(t)) {
PyErr_SetString(PyExc_TypeError,
"_p_resolveConflict: expected tuple or None for state");
return NULL;
}
if (PyTuple_GET_SIZE(t) == 2) {
/* A non-degenerate BTree. */
return merge_error(-1, -1, -1, 11);
}
/* We're in the one-bucket case. */
if (PyTuple_GET_SIZE(t) != 1) {
PyErr_SetString(PyExc_TypeError,
"_p_resolveConflict: expected 1- or 2-tuple for state");
return NULL;
}
t = PyTuple_GET_ITEM(t, 0);
if (! PyTuple_Check(t) || PyTuple_GET_SIZE(t) != 1) {
PyErr_SetString(PyExc_TypeError,
"_p_resolveConflict: expected 1-tuple containing "
"bucket state");
return NULL;
}
t = PyTuple_GET_ITEM(t, 0);
if (! PyTuple_Check(t)) {
PyErr_SetString(PyExc_TypeError,
"_p_resolveConflict: expected tuple for bucket state");
return NULL;
}
return t;
}
/* Tricky. The only kind of BTree conflict we can actually potentially
* resolve is the special case of a BTree containing a single bucket,
* in which case this becomes a fancy way of calling the bucket conflict
* resolution code.
*/
static PyObject *
BTree__p_resolveConflict(BTree *self, PyObject *args)
{
PyObject *s[3];
PyObject *x, *y, *z;
if (!PyArg_ParseTuple(args, "OOO", &x, &y, &z))
return NULL;
s[0] = get_bucket_state(x);
if (s[0] == NULL)
return NULL;
s[1] = get_bucket_state(y);
if (s[1] == NULL)
return NULL;
s[2] = get_bucket_state(z);
if (s[2] == NULL)
return NULL;
if (PyObject_IsInstance((PyObject *)self, (PyObject *)&BTreeType))
x = _bucket__p_resolveConflict(OBJECT(&BucketType), s);
else
x = _bucket__p_resolveConflict(OBJECT(&SetType), s);
if (x == NULL)
return NULL;
return Py_BuildValue("((N))", x);
}
#endif
/*
BTree_findRangeEnd -- Find one end, expressed as a bucket and
position, for a range search.
If low, return bucket and index of the smallest item >= key,
otherwise return bucket and index of the largest item <= key.
If exclude_equal, exact matches aren't acceptable; if one is found,
move right if low, or left if !low (this is for range searches exclusive
of an endpoint).
Return:
-1 Error; offset and bucket unchanged
0 Not found; offset and bucket unchanged
1 Correct bucket and offset stored; the caller owns a new reference
to the bucket.
Internal:
We do binary searches in BTree nodes downward, at each step following
C(i) where K(i) <= key < K(i+1). As always, K(i) <= C(i) < K(i+1) too.
(See Maintainer.txt for the meaning of that notation.) That eventually
leads to a bucket where we do Bucket_findRangeEnd. That usually works,
but there are two cases where it can fail to find the correct answer:
1. On a low search, we find a bucket with keys >= K(i), but that doesn't
imply there are keys in the bucket >= key. For example, suppose
a bucket has keys in 1..100, its successor's keys are in 200..300, and
we're doing a low search on 150. We'll end up in the first bucket,
but there are no keys >= 150 in it. K(i+1) > key, though, and all
the keys in C(i+1) >= K(i+1) > key, so the first key in the next
bucket (if any) is the correct result. This is easy to find by
following the bucket 'next' pointer.
2. On a high search, again that the keys in the bucket are >= K(i)
doesn't imply that any key in the bucket is <= key, but it's harder
for this to fail (and an earlier version of this routine didn't
catch it): if K(i) itself is in the bucket, it works (then
K(i) <= key is *a* key in the bucket that's in the desired range).
But when keys get deleted from buckets, they aren't also deleted from
BTree nodes, so there's no guarantee that K(i) is in the bucket.
For example, delete the smallest key S from some bucket, and S
remains in the interior BTree nodes. Do a high search for S, and
the BTree nodes direct the search to the bucket S used to be in,
but all keys remaining in that bucket are > S. The largest key in
the *preceding* bucket (if any) is < K(i), though, and K(i) <= key,
so the largest key in the preceding bucket is < key and so is the
proper result.
This is harder to get at efficiently, as buckets are linked only in
the increasing direction. While we're searching downward,
deepest_smaller is set to the node deepest in the tree where
we *could* have gone to the left of C(i). The rightmost bucket in
deepest_smaller's subtree is the bucket preceding the bucket we find
at first. This is clumsy to get at, but efficient.
*/
static int
BTree_findRangeEnd(BTree *self, PyObject *keyarg, int low, int exclude_equal,
Bucket **bucket, int *offset) {
Sized *deepest_smaller = NULL; /* last possibility to move left */
int deepest_smaller_is_btree = 0; /* Boolean; if false, it's a bucket */
Bucket *pbucket;
int self_got_rebound = 0; /* Boolean; when true, deactivate self */
int result = -1; /* Until proven innocent */
int i;
KEY_TYPE key;
int copied = 1;
COPY_KEY_FROM_ARG(key, keyarg, copied);
UNLESS (copied) return -1;
/* We don't need to: PER_USE_OR_RETURN(self, -1);
because the caller does. */
UNLESS (self->data && self->len) return 0;
/* Search downward until hitting a bucket, stored in pbucket. */
for (;;) {
Sized *pchild;
int pchild_is_btree;
BTREE_SEARCH(i, self, key, goto Done);
pchild = self->data[i].child;
pchild_is_btree = SameType_Check(self, pchild);
if (i) {
deepest_smaller = self->data[i-1].child;
deepest_smaller_is_btree = pchild_is_btree;
}
if (pchild_is_btree) {
if (self_got_rebound) {
PER_UNUSE(self);
}
self = BTREE(pchild);
self_got_rebound = 1;
PER_USE_OR_RETURN(self, -1);
}
else {
pbucket = BUCKET(pchild);
break;
}
}
/* Search the bucket for a suitable key. */
i = Bucket_findRangeEnd(pbucket, keyarg, low, exclude_equal, offset);
if (i < 0)
goto Done;
if (i > 0) {
Py_INCREF(pbucket);
*bucket = pbucket;
result = 1;
goto Done;
}
/* This may be one of the two difficult cases detailed in the comments. */
if (low) {
Bucket *next;
UNLESS(PER_USE(pbucket)) goto Done;
next = pbucket->next;
if (next) {
result = 1;
Py_INCREF(next);
*bucket = next;
*offset = 0;
}
else
result = 0;
PER_UNUSE(pbucket);
}
/* High-end search: if it's possible to go left, do so. */
else if (deepest_smaller) {
if (deepest_smaller_is_btree) {
UNLESS(PER_USE(deepest_smaller)) goto Done;
/* We own the reference this returns. */
pbucket = BTree_lastBucket(BTREE(deepest_smaller));
PER_UNUSE(deepest_smaller);
if (pbucket == NULL) goto Done; /* error */
}
else {
pbucket = BUCKET(deepest_smaller);
Py_INCREF(pbucket);
}
UNLESS(PER_USE(pbucket)) goto Done;
result = 1;
*bucket = pbucket; /* transfer ownership to caller */
*offset = pbucket->len - 1;
PER_UNUSE(pbucket);
}
else
result = 0; /* simply not found */
Done:
if (self_got_rebound) {
PER_UNUSE(self);
}
return result;
}
static PyObject *
BTree_maxminKey(BTree *self, PyObject *args, int min)
{
PyObject *key=0;
Bucket *bucket = NULL;
int offset, rc;
int empty_tree = 1;
UNLESS (PyArg_ParseTuple(args, "|O", &key)) return NULL;
UNLESS (PER_USE(self)) return NULL;
UNLESS (self->data && self->len) goto empty;
/* Find the range */
if (key)
{
if ((rc = BTree_findRangeEnd(self, key, min, 0, &bucket, &offset)) <= 0)
{
if (rc < 0) goto err;
empty_tree = 0;
goto empty;
}
PER_UNUSE(self);
UNLESS (PER_USE(bucket))
{
Py_DECREF(bucket);
return NULL;
}
}
else if (min)
{
bucket = self->firstbucket;
PER_UNUSE(self);
PER_USE_OR_RETURN(bucket, NULL);
Py_INCREF(bucket);
offset = 0;
}
else
{
bucket = BTree_lastBucket(self);
PER_UNUSE(self);
UNLESS (PER_USE(bucket))
{
Py_DECREF(bucket);
return NULL;
}
assert(bucket->len);
offset = bucket->len - 1;
}
COPY_KEY_TO_OBJECT(key, bucket->keys[offset]);
PER_UNUSE(bucket);
Py_DECREF(bucket);
return key;
empty:
PyErr_SetString(PyExc_ValueError,
empty_tree ? "empty tree" :
"no key satisfies the conditions");
err:
PER_UNUSE(self);
if (bucket)
{
PER_UNUSE(bucket);
Py_DECREF(bucket);
}
return NULL;
}
static PyObject *
BTree_minKey(BTree *self, PyObject *args)
{
return BTree_maxminKey(self, args, 1);
}
static PyObject *
BTree_maxKey(BTree *self, PyObject *args)
{
return BTree_maxminKey(self, args, 0);
}
/*
** BTree_rangeSearch
**
** Generates a BTreeItems object based on the two indexes passed in,
** being the range between them.
**
*/
static PyObject *
BTree_rangeSearch(BTree *self, PyObject *args, PyObject *kw, char type)
{
PyObject *min = Py_None;
PyObject *max = Py_None;
int excludemin = 0;
int excludemax = 0;
int rc;
Bucket *lowbucket = NULL;
Bucket *highbucket = NULL;
int lowoffset;
int highoffset;
PyObject *result;
if (args) {
if (! PyArg_ParseTupleAndKeywords(args, kw, "|OOii", search_keywords,
&min,
&max,
&excludemin,
&excludemax))
return NULL;
}
UNLESS (PER_USE(self)) return NULL;
UNLESS (self->data && self->len) goto empty;
/* Find the low range */
if (min != Py_None) {
if ((rc = BTree_findRangeEnd(self, min, 1, excludemin,
&lowbucket, &lowoffset)) <= 0) {
if (rc < 0) goto err;
goto empty;
}
}
else {
lowbucket = self->firstbucket;
lowoffset = 0;
if (excludemin) {
int bucketlen;
UNLESS (PER_USE(lowbucket)) goto err;
bucketlen = lowbucket->len;
PER_UNUSE(lowbucket);
if (bucketlen > 1)
lowoffset = 1;
else if (self->len < 2)
goto empty;
else { /* move to first item in next bucket */
Bucket *next;
UNLESS (PER_USE(lowbucket)) goto err;
next = lowbucket->next;
PER_UNUSE(lowbucket);
assert(next != NULL);
lowbucket = next;
/* and lowoffset is still 0 */
assert(lowoffset == 0);
}
}
Py_INCREF(lowbucket);
}
/* Find the high range */
if (max != Py_None) {
if ((rc = BTree_findRangeEnd(self, max, 0, excludemax,
&highbucket, &highoffset)) <= 0) {
Py_DECREF(lowbucket);
if (rc < 0) goto err;
goto empty;
}
}
else {
int bucketlen;
highbucket = BTree_lastBucket(self);
assert(highbucket != NULL); /* we know self isn't empty */
UNLESS (PER_USE(highbucket)) goto err_and_decref_buckets;
bucketlen = highbucket->len;
PER_UNUSE(highbucket);
highoffset = bucketlen - 1;
if (excludemax) {
if (highoffset > 0)
--highoffset;
else if (self->len < 2)
goto empty_and_decref_buckets;
else { /* move to last item of preceding bucket */
int status;
assert(highbucket != self->firstbucket);
Py_DECREF(highbucket);
status = PreviousBucket(&highbucket, self->firstbucket);
if (status < 0) {
Py_DECREF(lowbucket);
goto err;
}
assert(status > 0);
Py_INCREF(highbucket);
UNLESS (PER_USE(highbucket)) goto err_and_decref_buckets;
highoffset = highbucket->len - 1;
PER_UNUSE(highbucket);
}
}
assert(highoffset >= 0);
}
/* It's still possible that the range is empty, even if min < max. For
* example, if min=3 and max=4, and 3 and 4 aren't in the BTree, but 2 and
* 5 are, then the low position points to the 5 now and the high position
* points to the 2 now. They're not necessarily even in the same bucket,
* so there's no trick we can play with pointer compares to get out
* cheap in general.
*/
if (lowbucket == highbucket && lowoffset > highoffset)
goto empty_and_decref_buckets; /* definitely empty */
/* The buckets differ, or they're the same and the offsets show a non-
* empty range.
*/
if (min != Py_None && max != Py_None && /* both args user-supplied */
lowbucket != highbucket) /* and different buckets */ {
KEY_TYPE first;
KEY_TYPE last;
int cmp;
/* Have to check the hard way: see how the endpoints compare. */
UNLESS (PER_USE(lowbucket)) goto err_and_decref_buckets;
COPY_KEY(first, lowbucket->keys[lowoffset]);
PER_UNUSE(lowbucket);
UNLESS (PER_USE(highbucket)) goto err_and_decref_buckets;
COPY_KEY(last, highbucket->keys[highoffset]);
PER_UNUSE(highbucket);
TEST_KEY_SET_OR(cmp, first, last) goto err_and_decref_buckets;
if (cmp > 0) goto empty_and_decref_buckets;
}
PER_UNUSE(self);
result = newBTreeItems(type, lowbucket, lowoffset, highbucket, highoffset);
Py_DECREF(lowbucket);
Py_DECREF(highbucket);
return result;
err_and_decref_buckets:
Py_DECREF(lowbucket);
Py_DECREF(highbucket);
err:
PER_UNUSE(self);
return NULL;
empty_and_decref_buckets:
Py_DECREF(lowbucket);
Py_DECREF(highbucket);
empty:
PER_UNUSE(self);
return newBTreeItems(type, 0, 0, 0, 0);
}
/*
** BTree_keys
*/
static PyObject *
BTree_keys(BTree *self, PyObject *args, PyObject *kw)
{
return BTree_rangeSearch(self, args, kw, 'k');
}
/*
** BTree_values
*/
static PyObject *
BTree_values(BTree *self, PyObject *args, PyObject *kw)
{
return BTree_rangeSearch(self, args, kw, 'v');
}
/*
** BTree_items
*/
static PyObject *
BTree_items(BTree *self, PyObject *args, PyObject *kw)
{
return BTree_rangeSearch(self, args, kw, 'i');
}
static PyObject *
BTree_byValue(BTree *self, PyObject *omin)
{
PyObject *r=0, *o=0, *item=0;
VALUE_TYPE min;
VALUE_TYPE v;
int copied=1;
SetIteration it = {0, 0, 1};
UNLESS (PER_USE(self)) return NULL;
COPY_VALUE_FROM_ARG(min, omin, copied);
UNLESS(copied) return NULL;
UNLESS (r=PyList_New(0)) goto err;
it.set=BTree_rangeSearch(self, NULL, NULL, 'i');
UNLESS(it.set) goto err;
if (nextBTreeItems(&it) < 0) goto err;
while (it.position >= 0)
{
if (TEST_VALUE(it.value, min) >= 0)
{
UNLESS (item = PyTuple_New(2)) goto err;
COPY_KEY_TO_OBJECT(o, it.key);
UNLESS (o) goto err;
PyTuple_SET_ITEM(item, 1, o);
COPY_VALUE(v, it.value);
NORMALIZE_VALUE(v, min);
COPY_VALUE_TO_OBJECT(o, v);
DECREF_VALUE(v);
UNLESS (o) goto err;
PyTuple_SET_ITEM(item, 0, o);
if (PyList_Append(r, item) < 0) goto err;
Py_DECREF(item);
item = 0;
}
if (nextBTreeItems(&it) < 0) goto err;
}
item=PyObject_GetAttr(r,sort_str);
UNLESS (item) goto err;
ASSIGN(item, PyObject_CallObject(item, NULL));
UNLESS (item) goto err;
ASSIGN(item, PyObject_GetAttr(r, reverse_str));
UNLESS (item) goto err;
ASSIGN(item, PyObject_CallObject(item, NULL));
UNLESS (item) goto err;
Py_DECREF(item);
finiSetIteration(&it);
PER_UNUSE(self);
return r;
err:
PER_UNUSE(self);
Py_XDECREF(r);
finiSetIteration(&it);
Py_XDECREF(item);
return NULL;
}
/*
** BTree_getm
*/
static PyObject *
BTree_getm(BTree *self, PyObject *args)
{
PyObject *key, *d=Py_None, *r;
UNLESS (PyArg_ParseTuple(args, "O|O", &key, &d)) return NULL;
if ((r=_BTree_get(self, key, 0))) return r;
UNLESS (PyErr_ExceptionMatches(PyExc_KeyError)) return NULL;
PyErr_Clear();
Py_INCREF(d);
return d;
}
static PyObject *
BTree_has_key(BTree *self, PyObject *key)
{
return _BTree_get(self, key, 1);
}
static PyObject *
BTree_setdefault(BTree *self, PyObject *args)
{
PyObject *key;
PyObject *failobj; /* default */
PyObject *value; /* return value */
if (! PyArg_UnpackTuple(args, "setdefault", 2, 2, &key, &failobj))
return NULL;
value = _BTree_get(self, key, 0);
if (value != NULL)
return value;
/* The key isn't in the tree. If that's not due to a KeyError exception,
* pass back the unexpected exception.
*/
if (! PyErr_ExceptionMatches(PyExc_KeyError))
return NULL;
PyErr_Clear();
/* Associate `key` with `failobj` in the tree, and return `failobj`. */
value = failobj;
if (_BTree_set(self, key, failobj, 0, 0) < 0)
value = NULL;
Py_XINCREF(value);
return value;
}
/* forward declaration */
static Py_ssize_t
BTree_length_or_nonzero(BTree *self, int nonzero);
static PyObject *
BTree_pop(BTree *self, PyObject *args)
{
PyObject *key;
PyObject *failobj = NULL; /* default */
PyObject *value; /* return value */
if (! PyArg_UnpackTuple(args, "pop", 1, 2, &key, &failobj))
return NULL;
value = _BTree_get(self, key, 0);
if (value != NULL) {
/* Delete key and associated value. */
if (_BTree_set(self, key, NULL, 0, 0) < 0) {
Py_DECREF(value);
return NULL;;
}
return value;
}
/* The key isn't in the tree. If that's not due to a KeyError exception,
* pass back the unexpected exception.
*/
if (! PyErr_ExceptionMatches(PyExc_KeyError))
return NULL;
if (failobj != NULL) {
/* Clear the KeyError and return the explicit default. */
PyErr_Clear();
Py_INCREF(failobj);
return failobj;
}
/* No default given. The only difference in this case is the error
* message, which depends on whether the tree is empty.
*/
if (BTree_length_or_nonzero(self, 1) == 0) /* tree is empty */
PyErr_SetString(PyExc_KeyError, "pop(): BTree is empty");
return NULL;
}
/* Search BTree self for key. This is the sq_contains slot of the
* PySequenceMethods.
*
* Return:
* -1 error
* 0 not found
* 1 found
*/
static int
BTree_contains(BTree *self, PyObject *key)
{
PyObject *asobj = _BTree_get(self, key, 1);
int result = -1;
if (asobj != NULL) {
result = PyInt_AsLong(asobj) ? 1 : 0;
Py_DECREF(asobj);
}
return result;
}
static PyObject *
BTree_addUnique(BTree *self, PyObject *args)
{
int grew;
PyObject *key, *v;
UNLESS (PyArg_ParseTuple(args, "OO", &key, &v)) return NULL;
if ((grew=_BTree_set(self, key, v, 1, 0)) < 0) return NULL;
return PyInt_FromLong(grew);
}
/**************************************************************************/
/* Iterator support. */
/* A helper to build all the iterators for BTrees and TreeSets.
* If args is NULL, the iterator spans the entire structure. Else it's an
* argument tuple, with optional low and high arguments.
* kind is 'k', 'v' or 'i'.
* Returns a BTreeIter object, or NULL if error.
*/
static PyObject *
buildBTreeIter(BTree *self, PyObject *args, PyObject *kw, char kind)
{
BTreeIter *result = NULL;
BTreeItems *items = (BTreeItems *)BTree_rangeSearch(self, args, kw, kind);
if (items) {
result = BTreeIter_new(items);
Py_DECREF(items);
}
return (PyObject *)result;
}
/* The implementation of iter(BTree_or_TreeSet); the BTree tp_iter slot. */
static PyObject *
BTree_getiter(BTree *self)
{
return buildBTreeIter(self, NULL, NULL, 'k');
}
/* The implementation of BTree.iterkeys(). */
static PyObject *
BTree_iterkeys(BTree *self, PyObject *args, PyObject *kw)
{
return buildBTreeIter(self, args, kw, 'k');
}
/* The implementation of BTree.itervalues(). */
static PyObject *
BTree_itervalues(BTree *self, PyObject *args, PyObject *kw)
{
return buildBTreeIter(self, args, kw, 'v');
}
/* The implementation of BTree.iteritems(). */
static PyObject *
BTree_iteritems(BTree *self, PyObject *args, PyObject *kw)
{
return buildBTreeIter(self, args, kw, 'i');
}
/* End of iterator support. */
/* Caution: Even though the _firstbucket attribute is read-only, a program
could do arbitrary damage to the btree internals. For example, it could
call clear() on a bucket inside a BTree.
We need to decide if the convenience for inspecting BTrees is worth
the risk.
*/
static struct PyMemberDef BTree_members[] = {
{"_firstbucket", T_OBJECT, offsetof(BTree, firstbucket), RO},
{NULL}
};
static struct PyMethodDef BTree_methods[] = {
{"__getstate__", (PyCFunction) BTree_getstate, METH_NOARGS,
"__getstate__() -> state\n\n"
"Return the picklable state of the BTree."},
{"__setstate__", (PyCFunction) BTree_setstate, METH_O,
"__setstate__(state)\n\n"
"Set the state of the BTree."},
{"has_key", (PyCFunction) BTree_has_key, METH_O,
"has_key(key)\n\n"
"Return true if the BTree contains the given key."},
{"keys", (PyCFunction) BTree_keys, METH_KEYWORDS,
"keys([min, max]) -> list of keys\n\n"
"Returns the keys of the BTree. If min and max are supplied, only\n"
"keys greater than min and less than max are returned."},
{"values", (PyCFunction) BTree_values, METH_KEYWORDS,
"values([min, max]) -> list of values\n\n"
"Returns the values of the BTree. If min and max are supplied, only\n"
"values corresponding to keys greater than min and less than max are\n"
"returned."},
{"items", (PyCFunction) BTree_items, METH_KEYWORDS,
"items([min, max]) -> -- list of key, value pairs\n\n"
"Returns the items of the BTree. If min and max are supplied, only\n"
"items with keys greater than min and less than max are returned."},
{"byValue", (PyCFunction) BTree_byValue, METH_O,
"byValue(min) -> list of value, key pairs\n\n"
"Returns list of value, key pairs where the value is >= min. The\n"
"list is sorted by value. Note that items() returns keys in the\n"
"opposite order."},
{"get", (PyCFunction) BTree_getm, METH_VARARGS,
"get(key[, default=None]) -> Value for key or default\n\n"
"Return the value or the default if the key is not found."},
{"setdefault", (PyCFunction) BTree_setdefault, METH_VARARGS,
"D.setdefault(k, d) -> D.get(k, d), also set D[k]=d if k not in D.\n\n"
"Return the value like get() except that if key is missing, d is both\n"
"returned and inserted into the BTree as the value of k."},
{"pop", (PyCFunction) BTree_pop, METH_VARARGS,
"D.pop(k[, d]) -> v, remove key and return the corresponding value.\n\n"
"If key is not found, d is returned if given, otherwise KeyError\n"
"is raised."},
{"maxKey", (PyCFunction) BTree_maxKey, METH_VARARGS,
"maxKey([max]) -> key\n\n"
"Return the largest key in the BTree. If max is specified, return\n"
"the largest key <= max."},
{"minKey", (PyCFunction) BTree_minKey, METH_VARARGS,
"minKey([mi]) -> key\n\n"
"Return the smallest key in the BTree. If min is specified, return\n"
"the smallest key >= min."},
{"clear", (PyCFunction) BTree_clear, METH_NOARGS,
"clear()\n\nRemove all of the items from the BTree."},
{"insert", (PyCFunction)BTree_addUnique, METH_VARARGS,
"insert(key, value) -> 0 or 1\n\n"
"Add an item if the key is not already used. Return 1 if the item was\n"
"added, or 0 otherwise."},
{"update", (PyCFunction) Mapping_update, METH_O,
"update(collection)\n\n Add the items from the given collection."},
{"iterkeys", (PyCFunction) BTree_iterkeys, METH_KEYWORDS,
"B.iterkeys([min[,max]]) -> an iterator over the keys of B"},
{"itervalues", (PyCFunction) BTree_itervalues, METH_KEYWORDS,
"B.itervalues([min[,max]]) -> an iterator over the values of B"},
{"iteritems", (PyCFunction) BTree_iteritems, METH_KEYWORDS,
"B.iteritems([min[,max]]) -> an iterator over the (key, value) items of B"},
{"_check", (PyCFunction) BTree_check, METH_NOARGS,
"Perform sanity check on BTree, and raise exception if flawed."},
#ifdef PERSISTENT
{"_p_resolveConflict", (PyCFunction) BTree__p_resolveConflict,
METH_VARARGS,
"_p_resolveConflict() -- Reinitialize from a newly created copy"},
{"_p_deactivate", (PyCFunction) BTree__p_deactivate, METH_KEYWORDS,
"_p_deactivate()\n\nReinitialize from a newly created copy."},
#endif
{NULL, NULL}
};
static int
BTree_init(PyObject *self, PyObject *args, PyObject *kwds)
{
PyObject *v = NULL;
if (!PyArg_ParseTuple(args, "|O:" MOD_NAME_PREFIX "BTree", &v))
return -1;
if (v)
return update_from_seq(self, v);
else
return 0;
}
static void
BTree_dealloc(BTree *self)
{
if (self->state != cPersistent_GHOST_STATE)
_BTree_clear(self);
cPersistenceCAPI->pertype->tp_dealloc((PyObject *)self);
}
static int
BTree_traverse(BTree *self, visitproc visit, void *arg)
{
int err = 0;
int i, len;
#define VISIT(SLOT) \
if (SLOT) { \
err = visit((PyObject *)(SLOT), arg); \
if (err) \
goto Done; \
}
if (self->ob_type == &BTreeType)
assert(self->ob_type->tp_dictoffset == 0);
/* Call our base type's traverse function. Because BTrees are
* subclasses of Peristent, there must be one.
*/
err = cPersistenceCAPI->pertype->tp_traverse((PyObject *)self, visit, arg);
if (err)
goto Done;
/* If this is registered with the persistence system, cleaning up cycles
* is the database's problem. It would be horrid to unghostify BTree
* nodes here just to chase pointers every time gc runs.
*/
if (self->state == cPersistent_GHOST_STATE)
goto Done;
len = self->len;
#ifdef KEY_TYPE_IS_PYOBJECT
/* Keys are Python objects so need to be traversed. Note that the
* key 0 slot is unused and should not be traversed.
*/
for (i = 1; i < len; i++)
VISIT(self->data[i].key);
#endif
/* Children are always pointers, and child 0 is legit. */
for (i = 0; i < len; i++)
VISIT(self->data[i].child);
VISIT(self->firstbucket);
Done:
return err;
#undef VISIT
}
static int
BTree_tp_clear(BTree *self)
{
if (self->state != cPersistent_GHOST_STATE)
_BTree_clear(self);
return 0;
}
/*
* Return the number of elements in a BTree. nonzero is a Boolean, and
* when true requests just a non-empty/empty result. Testing for emptiness
* is efficient (constant-time). Getting the true length takes time
* proportional to the number of leaves (buckets).
*
* Return:
* When nonzero true:
* -1 error
* 0 empty
* 1 not empty
* When nonzero false (possibly expensive!):
* -1 error
* >= 0 number of elements.
*/
static Py_ssize_t
BTree_length_or_nonzero(BTree *self, int nonzero)
{
int result;
Bucket *b;
Bucket *next;
PER_USE_OR_RETURN(self, -1);
b = self->firstbucket;
PER_UNUSE(self);
if (nonzero)
return b != NULL;
result = 0;
while (b) {
PER_USE_OR_RETURN(b, -1);
result += b->len;
next = b->next;
PER_UNUSE(b);
b = next;
}
return result;
}
static Py_ssize_t
BTree_length(BTree *self)
{
return BTree_length_or_nonzero(self, 0);
}
static PyMappingMethods BTree_as_mapping = {
(lenfunc)BTree_length, /*mp_length*/
(binaryfunc)BTree_get, /*mp_subscript*/
(objobjargproc)BTree_setitem, /*mp_ass_subscript*/
};
static PySequenceMethods BTree_as_sequence = {
(lenfunc)0, /* sq_length */
(binaryfunc)0, /* sq_concat */
(ssizeargfunc)0, /* sq_repeat */
(ssizeargfunc)0, /* sq_item */
(ssizessizeargfunc)0, /* sq_slice */
(ssizeobjargproc)0, /* sq_ass_item */
(ssizessizeobjargproc)0, /* sq_ass_slice */
(objobjproc)BTree_contains, /* sq_contains */
0, /* sq_inplace_concat */
0, /* sq_inplace_repeat */
};
static Py_ssize_t
BTree_nonzero(BTree *self)
{
return BTree_length_or_nonzero(self, 1);
}
static PyNumberMethods BTree_as_number_for_nonzero = {
0,0,0,0,0,0,0,0,0,0,
(inquiry)BTree_nonzero};
static PyTypeObject BTreeType = {
PyObject_HEAD_INIT(NULL) /* PyPersist_Type */
0, /* ob_size */
MODULE_NAME MOD_NAME_PREFIX "BTree",/* tp_name */
sizeof(BTree), /* tp_basicsize */
0, /* tp_itemsize */
(destructor)BTree_dealloc, /* tp_dealloc */
0, /* tp_print */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_compare */
0, /* tp_repr */
&BTree_as_number_for_nonzero, /* tp_as_number */
&BTree_as_sequence, /* tp_as_sequence */
&BTree_as_mapping, /* tp_as_mapping */
0, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
0, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
Py_TPFLAGS_BASETYPE, /* tp_flags */
0, /* tp_doc */
(traverseproc)BTree_traverse, /* tp_traverse */
(inquiry)BTree_tp_clear, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
(getiterfunc)BTree_getiter, /* tp_iter */
0, /* tp_iternext */
BTree_methods, /* tp_methods */
BTree_members, /* tp_members */
0, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
BTree_init, /* tp_init */
0, /* tp_alloc */
0, /*PyType_GenericNew,*/ /* tp_new */
};
BTrees/BucketTemplate.c 0 → 100644
View file @ 703fc7e7
/*****************************************************************************
Copyright (c) 2001, 2002 Zope Foundation and Contributors.
All Rights Reserved.
This software is subject to the provisions of the Zope Public License,
Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
****************************************************************************/
#define BUCKETTEMPLATE_C "$Id$\n"
/* Use BUCKET_SEARCH to find the index at which a key belongs.
* INDEX An int lvalue to hold the index i such that KEY belongs at
* SELF->keys[i]. Note that this will equal SELF->len if KEY
* is larger than the bucket's largest key. Else it's the
* smallest i such that SELF->keys[i] >= KEY.
* ABSENT An int lvalue to hold a Boolean result, true (!= 0) if the
* key is absent, false (== 0) if the key is at INDEX.
* SELF A pointer to a Bucket node.
* KEY The key you're looking for, of type KEY_TYPE.
* ONERROR What to do if key comparison raises an exception; for example,
* perhaps 'return NULL'.
*
* See Maintainer.txt for discussion: this is optimized in subtle ways.
* It's recommended that you call this at the start of a routine, waiting
* to check for self->len == 0 after (if an empty bucket is special in
* context; INDEX becomes 0 and ABSENT becomes true if this macro is run
* with an empty SELF, and that may be all the invoker needs to know).
*/
#define BUCKET_SEARCH(INDEX, ABSENT, SELF, KEY, ONERROR) { \
int _lo = 0; \
int _hi = (SELF)->len; \
int _i; \
int _cmp = 1; \
for (_i = _hi >> 1; _lo < _hi; _i = (_lo + _hi) >> 1) { \
TEST_KEY_SET_OR(_cmp, (SELF)->keys[_i], (KEY)) \
ONERROR; \
if (_cmp < 0) _lo = _i + 1; \
else if (_cmp == 0) break; \
else _hi = _i; \
} \
(INDEX) = _i; \
(ABSENT) = _cmp; \
}
/*
** _bucket_get
**
** Search a bucket for a given key.
**
** Arguments
** self The bucket
** keyarg The key to look for
** has_key Boolean; if true, return a true/false result; else return
** the value associated with the key.
**
** Return
** If has_key:
** Returns the Python int 0 if the key is absent, else returns
** has_key itself as a Python int. A BTree caller generally passes
** the depth of the bucket for has_key, so a true result returns
** the bucket depth then.
** Note that has_key should be true when searching set buckets.
** If not has_key:
** If the key is present, returns the associated value, and the
** caller owns the reference. Else returns NULL and sets KeyError.
** Whether or not has_key:
** If a comparison sets an exception, returns NULL.
*/
static PyObject *
_bucket_get(Bucket *self, PyObject *keyarg, int has_key)
{
int i, cmp;
KEY_TYPE key;
PyObject *r = NULL;
int copied = 1;
COPY_KEY_FROM_ARG(key, keyarg, copied);
UNLESS (copied) return NULL;
UNLESS (PER_USE(self)) return NULL;
BUCKET_SEARCH(i, cmp, self, key, goto Done);
if (has_key)
r = PyInt_FromLong(cmp ? 0 : has_key);
else {
if (cmp == 0) {
COPY_VALUE_TO_OBJECT(r, self->values[i]);
}
else
PyErr_SetObject(PyExc_KeyError, keyarg);
}
Done:
PER_UNUSE(self);
return r;
}
static PyObject *
bucket_getitem(Bucket *self, PyObject *key)
{
return _bucket_get(self, key, 0);
}
/*
** Bucket_grow
**
** Resize a bucket.
**
** Arguments: self The bucket.
** newsize The new maximum capacity. If < 0, double the
** current size unless the bucket is currently empty,
** in which case use MIN_BUCKET_ALLOC.
** noval Boolean; if true, allocate only key space and not
** value space
**
** Returns: -1 on error, and MemoryError exception is set
** 0 on success
*/
static int
Bucket_grow(Bucket *self, int newsize, int noval)
{
KEY_TYPE *keys;
VALUE_TYPE *values;
if (self->size) {
if (newsize < 0)
newsize = self->size * 2;
if (newsize < 0) /* int overflow */
goto Overflow;
UNLESS (keys = BTree_Realloc(self->keys, sizeof(KEY_TYPE) * newsize))
return -1;
UNLESS (noval) {
values = BTree_Realloc(self->values, sizeof(VALUE_TYPE) * newsize);
if (values == NULL) {
free(keys);
return -1;
}
self->values = values;
}
self->keys = keys;
}
else {
if (newsize < 0)
newsize = MIN_BUCKET_ALLOC;
UNLESS (self->keys = BTree_Malloc(sizeof(KEY_TYPE) * newsize))
return -1;
UNLESS (noval) {
self->values = BTree_Malloc(sizeof(VALUE_TYPE) * newsize);
if (self->values == NULL) {
free(self->keys);
self->keys = NULL;
return -1;
}
}
}
self->size = newsize;
return 0;
Overflow:
PyErr_NoMemory();
return -1;
}
/* So far, bucket_append is called only by multiunion_m(), so is called
* only when MULTI_INT_UNION is defined. Flavors of BTree/Bucket that
* don't support MULTI_INT_UNION don't call bucket_append (yet), and
* gcc complains if bucket_append is compiled in those cases. So only
* compile bucket_append if it's going to be used.
*/
#ifdef MULTI_INT_UNION
/*
* Append a slice of the "from" bucket to self.
*
* self Append (at least keys) to this bucket. self must be activated
* upon entry, and remains activated at exit. If copyValues
* is true, self must be empty or already have a non-NULL values
* pointer. self's access and modification times aren't updated.
* from The bucket from which to take keys, and possibly values. from
* must be activated upon entry, and remains activated at exit.
* If copyValues is true, from must have a non-NULL values
* pointer. self and from must not be the same. from's access
* time isn't updated.
* i, n The slice from[i : i+n] is appended to self. Must have
* i >= 0, n > 0 and i+n <= from->len.
* copyValues Boolean. If true, copy values from the slice as well as keys.
* In this case, from must have a non-NULL values pointer, and
* self must too (unless self is empty, in which case a values
* vector will be allocated for it).
* overallocate Boolean. If self doesn't have enough room upon entry to hold
* all the appended stuff, then if overallocate is false exactly
* enough room will be allocated to hold the new stuff, else if
* overallocate is true an excess will be allocated. overallocate
* may be a good idea if you expect to append more stuff to self
* later; else overallocate should be false.
*
* CAUTION: If self is empty upon entry (self->size == 0), and copyValues is
* false, then no space for values will get allocated. This can be a trap if
* the caller intends to copy values itself.
*
* Return
* -1 Error.
* 0 OK.
*/
static int
bucket_append(Bucket *self, Bucket *from, int i, int n,
int copyValues, int overallocate)
{
int newlen;
assert(self && from && self != from);
assert(i >= 0);
assert(n > 0);
assert(i+n <= from->len);
/* Make room. */
newlen = self->len + n;
if (newlen > self->size) {
int newsize = newlen;
if (overallocate) /* boost by 25% -- pretty arbitrary */
newsize += newsize >> 2;
if (Bucket_grow(self, newsize, ! copyValues) < 0)
return -1;
}
assert(newlen <= self->size);
/* Copy stuff. */
memcpy(self->keys + self->len, from->keys + i, n * sizeof(KEY_TYPE));
if (copyValues) {
assert(self->values);
assert(from->values);
memcpy(self->values + self->len, from->values + i,
n * sizeof(VALUE_TYPE));
}
self->len = newlen;
/* Bump refcounts. */
#ifdef KEY_TYPE_IS_PYOBJECT
{
int j;
PyObject **p = from->keys + i;
for (j = 0; j < n; ++j, ++p) {
Py_INCREF(*p);
}
}
#endif
#ifdef VALUE_TYPE_IS_PYOBJECT
if (copyValues) {
int j;
PyObject **p = from->values + i;
for (j = 0; j < n; ++j, ++p) {
Py_INCREF(*p);
}
}
#endif
return 0;
}
#endif /* MULTI_INT_UNION */
/*
** _bucket_set: Assign a value to a key in a bucket, delete a key+value
** pair, or just insert a key.
**
** Arguments
** self The bucket
** keyarg The key to look for
** v The value to associate with key; NULL means delete the key.
** If NULL, it's an error (KeyError) if the key isn't present.
** Note that if this is a set bucket, and you want to insert
** a new set element, v must be non-NULL although its exact
** value will be ignored. Passing Py_None is good for this.
** unique Boolean; when true, don't replace the value if the key is
** already present.
** noval Boolean; when true, operate on keys only (ignore values)
** changed ignored on input
**
** Return
** -1 on error
** 0 on success and the # of bucket entries didn't change
** 1 on success and the # of bucket entries did change
** *changed If non-NULL, set to 1 on any mutation of the bucket.
*/
static int
_bucket_set(Bucket *self, PyObject *keyarg, PyObject *v,
int unique, int noval, int *changed)
{
int i, cmp;
KEY_TYPE key;
/* Subtle: there may or may not be a value. If there is, we need to
* check its type early, so that in case of error we can get out before
* mutating the bucket. But because value isn't used on all paths, if
* we don't initialize value then gcc gives a nuisance complaint that
* value may be used initialized (it can't be, but gcc doesn't know
* that). So we initialize it. However, VALUE_TYPE can be various types,
* including int, PyObject*, and char[6], so it's a puzzle to spell
* initialization. It so happens that {0} is a valid initializer for all
* these types.
*/
VALUE_TYPE value = {0}; /* squash nuisance warning */
int result = -1; /* until proven innocent */
int copied = 1;
COPY_KEY_FROM_ARG(key, keyarg, copied);
UNLESS(copied) return -1;
/* Copy the value early (if needed), so that in case of error a
* pile of bucket mutations don't need to be undone.
*/
if (v && !noval) {
COPY_VALUE_FROM_ARG(value, v, copied);
UNLESS(copied) return -1;
}
UNLESS (PER_USE(self)) return -1;
BUCKET_SEARCH(i, cmp, self, key, goto Done);
if (cmp == 0) {
/* The key exists, at index i. */
if (v) {
/* The key exists at index i, and there's a new value.
* If unique, we're not supposed to replace it. If noval, or this
* is a set bucket (self->values is NULL), there's nothing to do.
*/
if (unique || noval || self->values == NULL) {
result = 0;
goto Done;
}
/* The key exists at index i, and we need to replace the value. */
#ifdef VALUE_SAME
/* short-circuit if no change */
if (VALUE_SAME(self->values[i], value)) {
result = 0;
goto Done;
}
#endif
if (changed)
*changed = 1;
DECREF_VALUE(self->values[i]);
COPY_VALUE(self->values[i], value);
INCREF_VALUE(self->values[i]);
if (PER_CHANGED(self) >= 0)
result = 0;
goto Done;
}
/* The key exists at index i, and should be deleted. */
DECREF_KEY(self->keys[i]);
self->len--;
if (i < self->len)
memmove(self->keys + i, self->keys + i+1,
sizeof(KEY_TYPE)*(self->len - i));
if (self->values) {
DECREF_VALUE(self->values[i]);
if (i < self->len)
memmove(self->values + i, self->values + i+1,
sizeof(VALUE_TYPE)*(self->len - i));
}
if (! self->len) {
self->size = 0;
free(self->keys);
self->keys = NULL;
if (self->values) {
free(self->values);
self->values = NULL;
}
}
if (changed)
*changed = 1;
if (PER_CHANGED(self) >= 0)
result = 1;
goto Done;
}
/* The key doesn't exist, and belongs at index i. */
if (!v) {
/* Can't delete a non-existent key. */
PyErr_SetObject(PyExc_KeyError, keyarg);
goto Done;
}
/* The key doesn't exist and should be inserted at index i. */
if (self->len == self->size && Bucket_grow(self, -1, noval) < 0)
goto Done;
if (self->len > i) {
memmove(self->keys + i + 1, self->keys + i,
sizeof(KEY_TYPE) * (self->len - i));
if (self->values) {
memmove(self->values + i + 1, self->values + i,
sizeof(VALUE_TYPE) * (self->len - i));
}
}
COPY_KEY(self->keys[i], key);
INCREF_KEY(self->keys[i]);
if (! noval) {
COPY_VALUE(self->values[i], value);
INCREF_VALUE(self->values[i]);
}
self->len++;
if (changed)
*changed = 1;
if (PER_CHANGED(self) >= 0)
result = 1;
Done:
PER_UNUSE(self);
return result;
}
/*
** bucket_setitem
**
** wrapper for _bucket_setitem (eliminates +1 return code)
**
** Arguments: self The bucket
** key The key to insert under
** v The value to insert
**
** Returns 0 on success
** -1 on failure
*/
static int
bucket_setitem(Bucket *self, PyObject *key, PyObject *v)
{
if (_bucket_set(self, key, v, 0, 0, 0) < 0)
return -1;
return 0;
}
/**
** Accepts a sequence of 2-tuples, or any object with an items()
** method that returns an iterable object producing 2-tuples.
*/
static int
update_from_seq(PyObject *map, PyObject *seq)
{
PyObject *iter, *o, *k, *v;
int err = -1;
/* One path creates a new seq object. The other path has an
INCREF of the seq argument. So seq must always be DECREFed on
the way out.
*/
/* Use items() if it's not a sequence. Alas, PySequence_Check()
* returns true for a PeristentMapping or PersistentDict, and we
* want to use items() in those cases too.
*/
if (!PySequence_Check(seq) || /* or it "looks like a dict" */
PyObject_HasAttrString(seq, "iteritems")) {
PyObject *items;
items = PyObject_GetAttrString(seq, "items");
if (items == NULL)
return -1;
seq = PyObject_CallObject(items, NULL);
Py_DECREF(items);
if (seq == NULL)
return -1;
}
else
Py_INCREF(seq);
iter = PyObject_GetIter(seq);
if (iter == NULL)
goto err;
while (1) {
o = PyIter_Next(iter);
if (o == NULL) {
if (PyErr_Occurred())
goto err;
else
break;
}
if (!PyTuple_Check(o) || PyTuple_GET_SIZE(o) != 2) {
Py_DECREF(o);
PyErr_SetString(PyExc_TypeError,
"Sequence must contain 2-item tuples");
goto err;
}
k = PyTuple_GET_ITEM(o, 0);
v = PyTuple_GET_ITEM(o, 1);
if (PyObject_SetItem(map, k, v) < 0) {
Py_DECREF(o);
goto err;
}
Py_DECREF(o);
}
err = 0;
err:
Py_DECREF(iter);
Py_DECREF(seq);
return err;
}
static PyObject *
Mapping_update(PyObject *self, PyObject *seq)
{
if (update_from_seq(self, seq) < 0)
return NULL;
Py_INCREF(Py_None);
return Py_None;
}
/*
** bucket_split
**
** Splits one bucket into two
**
** Arguments: self The bucket
** index the index of the key to split at (O.O.B use midpoint)
** next the new bucket to split into
**
** Returns: 0 on success
** -1 on failure
*/
static int
bucket_split(Bucket *self, int index, Bucket *next)
{
int next_size;
ASSERT(self->len > 1, "split of empty bucket", -1);
if (index < 0 || index >= self->len)
index = self->len / 2;
next_size = self->len - index;
next->keys = BTree_Malloc(sizeof(KEY_TYPE) * next_size);
if (!next->keys)
return -1;
memcpy(next->keys, self->keys + index, sizeof(KEY_TYPE) * next_size);
if (self->values) {
next->values = BTree_Malloc(sizeof(VALUE_TYPE) * next_size);
if (!next->values) {
free(next->keys);
next->keys = NULL;
return -1;
}
memcpy(next->values, self->values + index,
sizeof(VALUE_TYPE) * next_size);
}
next->size = next_size;
next->len = next_size;
self->len = index;
next->next = self->next;
Py_INCREF(next);
self->next = next;
if (PER_CHANGED(self) < 0)
return -1;
return 0;
}
/* Set self->next to self->next->next, i.e. unlink self's successor from
* the chain.
*
* Return:
* -1 error
* 0 OK
*/
static int
Bucket_deleteNextBucket(Bucket *self)
{
int result = -1; /* until proven innocent */
Bucket *successor;
PER_USE_OR_RETURN(self, -1);
successor = self->next;
if (successor) {
Bucket *next;
/* Before: self -> successor -> next
* After: self --------------> next
*/
UNLESS (PER_USE(successor)) goto Done;
next = successor->next;
PER_UNUSE(successor);
Py_XINCREF(next); /* it may be NULL, of course */
self->next = next;
Py_DECREF(successor);
if (PER_CHANGED(self) < 0)
goto Done;
}
result = 0;
Done:
PER_UNUSE(self);
return result;
}
/*
Bucket_findRangeEnd -- Find the index of a range endpoint
(possibly) contained in a bucket.
Arguments: self The bucket
keyarg The key to match against
low Boolean; true for low end of range, false for high
exclude_equal Boolean; if true, don't accept an exact match,
and if there is one then move right if low and
left if !low.
offset The output offset
If low true, *offset <- index of the smallest item >= key,
if low false the index of the largest item <= key. In either case, if there
is no such index, *offset is left alone and 0 is returned.
Return:
0 No suitable index exists; *offset has not been changed
1 The correct index was stored into *offset
-1 Error
Example: Suppose the keys are [2, 4], and exclude_equal is false. Searching
for 2 sets *offset to 0 and returns 1 regardless of low. Searching for 4
sets *offset to 1 and returns 1 regardless of low.
Searching for 1:
If low true, sets *offset to 0, returns 1.
If low false, returns 0.
Searching for 3:
If low true, sets *offset to 1, returns 1.
If low false, sets *offset to 0, returns 1.
Searching for 5:
If low true, returns 0.
If low false, sets *offset to 1, returns 1.
The 1, 3 and 5 examples are the same when exclude_equal is true.
*/
static int
Bucket_findRangeEnd(Bucket *self, PyObject *keyarg, int low, int exclude_equal,
int *offset)
{
int i, cmp;
int result = -1; /* until proven innocent */
KEY_TYPE key;
int copied = 1;
COPY_KEY_FROM_ARG(key, keyarg, copied);
UNLESS (copied) return -1;
UNLESS (PER_USE(self)) return -1;
BUCKET_SEARCH(i, cmp, self, key, goto Done);
if (cmp == 0) {
/* exact match at index i */
if (exclude_equal) {
/* but we don't want an exact match */
if (low)
++i;
else
--i;
}
}
/* Else keys[i-1] < key < keys[i], picturing infinities at OOB indices,
* and i has the smallest item > key, which is correct for low.
*/
else if (! low)
/* i-1 has the largest item < key (unless i-1 is 0OB) */
--i;
result = 0 <= i && i < self->len;
if (result)
*offset = i;
Done:
PER_UNUSE(self);
return result;
}
static PyObject *
Bucket_maxminKey(Bucket *self, PyObject *args, int min)
{
PyObject *key=0;
int rc, offset = 0;
int empty_bucket = 1;
if (args && ! PyArg_ParseTuple(args, "|O", &key)) return NULL;
PER_USE_OR_RETURN(self, NULL);
UNLESS (self->len) goto empty;
/* Find the low range */
if (key)
{
if ((rc = Bucket_findRangeEnd(self, key, min, 0, &offset)) <= 0)
{
if (rc < 0) return NULL;
empty_bucket = 0;
goto empty;
}
}
else if (min) offset = 0;
else offset = self->len -1;
COPY_KEY_TO_OBJECT(key, self->keys[offset]);
PER_UNUSE(self);
return key;
empty:
PyErr_SetString(PyExc_ValueError,
empty_bucket ? "empty bucket" :
"no key satisfies the conditions");
PER_UNUSE(self);
return NULL;
}
static PyObject *
Bucket_minKey(Bucket *self, PyObject *args)
{
return Bucket_maxminKey(self, args, 1);
}
static PyObject *
Bucket_maxKey(Bucket *self, PyObject *args)
{
return Bucket_maxminKey(self, args, 0);
}
static int
Bucket_rangeSearch(Bucket *self, PyObject *args, PyObject *kw,
int *low, int *high)
{
PyObject *min = Py_None;
PyObject *max = Py_None;
int excludemin = 0;
int excludemax = 0;
int rc;
if (args) {
if (! PyArg_ParseTupleAndKeywords(args, kw, "|OOii", search_keywords,
&min,
&max,
&excludemin,
&excludemax))
return -1;
}
UNLESS (self->len) goto empty;
/* Find the low range */
if (min != Py_None) {
rc = Bucket_findRangeEnd(self, min, 1, excludemin, low);
if (rc < 0)
return -1;
if (rc == 0)
goto empty;
}
else {
*low = 0;
if (excludemin) {
if (self->len < 2)
goto empty;
++*low;
}
}
/* Find the high range */
if (max != Py_None) {
rc = Bucket_findRangeEnd(self, max, 0, excludemax, high);
if (rc < 0)
return -1;
if (rc == 0)
goto empty;
}
else {
*high = self->len - 1;
if (excludemax) {
if (self->len < 2)
goto empty;
--*high;
}
}
/* If min < max to begin with, it's quite possible that low > high now. */
if (*low <= *high)
return 0;
empty:
*low = 0;
*high = -1;
return 0;
}
/*
** bucket_keys
**
** Generate a list of all keys in the bucket
**
** Arguments: self The Bucket
** args (unused)
**
** Returns: list of bucket keys
*/
static PyObject *
bucket_keys(Bucket *self, PyObject *args, PyObject *kw)
{
PyObject *r = NULL, *key;
int i, low, high;
PER_USE_OR_RETURN(self, NULL);
if (Bucket_rangeSearch(self, args, kw, &low, &high) < 0)
goto err;
r = PyList_New(high-low+1);
if (r == NULL)
goto err;
for (i=low; i <= high; i++) {
COPY_KEY_TO_OBJECT(key, self->keys[i]);
if (PyList_SetItem(r, i-low , key) < 0)
goto err;
}
PER_UNUSE(self);
return r;
err:
PER_UNUSE(self);
Py_XDECREF(r);
return NULL;
}
/*
** bucket_values
**
** Generate a list of all values in the bucket
**
** Arguments: self The Bucket
** args (unused)
**
** Returns list of values
*/
static PyObject *
bucket_values(Bucket *self, PyObject *args, PyObject *kw)
{
PyObject *r=0, *v;
int i, low, high;
PER_USE_OR_RETURN(self, NULL);
if (Bucket_rangeSearch(self, args, kw, &low, &high) < 0) goto err;
UNLESS (r=PyList_New(high-low+1)) goto err;
for (i=low; i <= high; i++)
{
COPY_VALUE_TO_OBJECT(v, self->values[i]);
UNLESS (v) goto err;
if (PyList_SetItem(r, i-low, v) < 0) goto err;
}
PER_UNUSE(self);
return r;
err:
PER_UNUSE(self);
Py_XDECREF(r);
return NULL;
}
/*
** bucket_items
**
** Returns a list of all items in a bucket
**
** Arguments: self The Bucket
** args (unused)
**
** Returns: list of all items in the bucket
*/
static PyObject *
bucket_items(Bucket *self, PyObject *args, PyObject *kw)
{
PyObject *r=0, *o=0, *item=0;
int i, low, high;
PER_USE_OR_RETURN(self, NULL);
if (Bucket_rangeSearch(self, args, kw, &low, &high) < 0) goto err;
UNLESS (r=PyList_New(high-low+1)) goto err;
for (i=low; i <= high; i++)
{
UNLESS (item = PyTuple_New(2)) goto err;
COPY_KEY_TO_OBJECT(o, self->keys[i]);
UNLESS (o) goto err;
PyTuple_SET_ITEM(item, 0, o);
COPY_VALUE_TO_OBJECT(o, self->values[i]);
UNLESS (o) goto err;
PyTuple_SET_ITEM(item, 1, o);
if (PyList_SetItem(r, i-low, item) < 0) goto err;
item = 0;
}
PER_UNUSE(self);
return r;
err:
PER_UNUSE(self);
Py_XDECREF(r);
Py_XDECREF(item);
return NULL;
}
static PyObject *
bucket_byValue(Bucket *self, PyObject *omin)
{
PyObject *r=0, *o=0, *item=0;
VALUE_TYPE min;
VALUE_TYPE v;
int i, l, copied=1;
PER_USE_OR_RETURN(self, NULL);
COPY_VALUE_FROM_ARG(min, omin, copied);
UNLESS(copied) return NULL;
for (i=0, l=0; i < self->len; i++)
if (TEST_VALUE(self->values[i], min) >= 0)
l++;
UNLESS (r=PyList_New(l)) goto err;
for (i=0, l=0; i < self->len; i++)
{
if (TEST_VALUE(self->values[i], min) < 0) continue;
UNLESS (item = PyTuple_New(2)) goto err;
COPY_KEY_TO_OBJECT(o, self->keys[i]);
UNLESS (o) goto err;
PyTuple_SET_ITEM(item, 1, o);
COPY_VALUE(v, self->values[i]);
NORMALIZE_VALUE(v, min);
COPY_VALUE_TO_OBJECT(o, v);
DECREF_VALUE(v);
UNLESS (o) goto err;
PyTuple_SET_ITEM(item, 0, o);
if (PyList_SetItem(r, l, item) < 0) goto err;
l++;
item = 0;
}
item=PyObject_GetAttr(r,sort_str);
UNLESS (item) goto err;
ASSIGN(item, PyObject_CallObject(item, NULL));
UNLESS (item) goto err;
ASSIGN(item, PyObject_GetAttr(r, reverse_str));
UNLESS (item) goto err;
ASSIGN(item, PyObject_CallObject(item, NULL));
UNLESS (item) goto err;
Py_DECREF(item);
PER_UNUSE(self);
return r;
err:
PER_UNUSE(self);
Py_XDECREF(r);
Py_XDECREF(item);
return NULL;
}
static int
_bucket_clear(Bucket *self)
{
const int len = self->len;
/* Don't declare i at this level. If neither keys nor values are
* PyObject*, i won't be referenced, and you'll get a nuisance compiler
* wng for declaring it here.
*/
self->len = self->size = 0;
if (self->next) {
Py_DECREF(self->next);
self->next = NULL;
}
/* Silence compiler warning about unused variable len for the case
when neither key nor value is an object, i.e. II. */
(void)len;
if (self->keys) {
#ifdef KEY_TYPE_IS_PYOBJECT
int i;
for (i = 0; i < len; ++i)
DECREF_KEY(self->keys[i]);
#endif
free(self->keys);
self->keys = NULL;
}
if (self->values) {
#ifdef VALUE_TYPE_IS_PYOBJECT
int i;
for (i = 0; i < len; ++i)
DECREF_VALUE(self->values[i]);
#endif
free(self->values);
self->values = NULL;
}
return 0;
}
#ifdef PERSISTENT
static PyObject *
bucket__p_deactivate(Bucket *self, PyObject *args, PyObject *keywords)
{
int ghostify = 1;
PyObject *force = NULL;
if (args && PyTuple_GET_SIZE(args) > 0) {
PyErr_SetString(PyExc_TypeError,
"_p_deactivate takes no positional arguments");
return NULL;
}
if (keywords) {
int size = PyDict_Size(keywords);
force = PyDict_GetItemString(keywords, "force");
if (force)
size--;
if (size) {
PyErr_SetString(PyExc_TypeError,
"_p_deactivate only accepts keyword arg force");
return NULL;
}
}
if (self->jar && self->oid) {
ghostify = self->state == cPersistent_UPTODATE_STATE;
if (!ghostify && force) {
if (PyObject_IsTrue(force))
ghostify = 1;
if (PyErr_Occurred())
return NULL;
}
if (ghostify) {
if (_bucket_clear(self) < 0)
return NULL;
PER_GHOSTIFY(self);
}
}
Py_INCREF(Py_None);
return Py_None;
}
#endif
static PyObject *
bucket_clear(Bucket *self, PyObject *args)
{
PER_USE_OR_RETURN(self, NULL);
if (self->len) {
if (_bucket_clear(self) < 0)
return NULL;
if (PER_CHANGED(self) < 0)
goto err;
}
PER_UNUSE(self);
Py_INCREF(Py_None);
return Py_None;
err:
PER_UNUSE(self);
return NULL;
}
/*
* Return:
*
* For a set bucket (self->values is NULL), a one-tuple or two-tuple. The
* first element is a tuple of keys, of length self->len. The second element
* is the next bucket, present if and only if next is non-NULL:
*
* (
* (keys[0], keys[1], ..., keys[len-1]),
* <self->next iff non-NULL>
* )
*
* For a mapping bucket (self->values is not NULL), a one-tuple or two-tuple.
* The first element is a tuple interleaving keys and values, of length
* 2 * self->len. The second element is the next bucket, present iff next is
* non-NULL:
*
* (
* (keys[0], values[0], keys[1], values[1], ...,
* keys[len-1], values[len-1]),
* <self->next iff non-NULL>
* )
*/
static PyObject *
bucket_getstate(Bucket *self)
{
PyObject *o = NULL, *items = NULL, *state;
int i, len, l;
PER_USE_OR_RETURN(self, NULL);
len = self->len;
if (self->values) { /* Bucket */
items = PyTuple_New(len * 2);
if (items == NULL)
goto err;
for (i = 0, l = 0; i < len; i++) {
COPY_KEY_TO_OBJECT(o, self->keys[i]);
if (o == NULL)
goto err;
PyTuple_SET_ITEM(items, l, o);
l++;
COPY_VALUE_TO_OBJECT(o, self->values[i]);
if (o == NULL)
goto err;
PyTuple_SET_ITEM(items, l, o);
l++;
}
} else { /* Set */
items = PyTuple_New(len);
if (items == NULL)
goto err;
for (i = 0; i < len; i++) {
COPY_KEY_TO_OBJECT(o, self->keys[i]);
if (o == NULL)
goto err;
PyTuple_SET_ITEM(items, i, o);
}
}
if (self->next)
state = Py_BuildValue("OO", items, self->next);
else
state = Py_BuildValue("(O)", items);
Py_DECREF(items);
PER_UNUSE(self);
return state;
err:
PER_UNUSE(self);
Py_XDECREF(items);
return NULL;
}
static int
_bucket_setstate(Bucket *self, PyObject *state)
{
PyObject *k, *v, *items;
Bucket *next = NULL;
int i, l, len, copied=1;
KEY_TYPE *keys;
VALUE_TYPE *values;
if (!PyArg_ParseTuple(state, "O|O:__setstate__", &items, &next))
return -1;
if (!PyTuple_Check(items)) {
PyErr_SetString(PyExc_TypeError,
"tuple required for first state element");
return -1;
}
len = PyTuple_Size(items);
if (len < 0)
return -1;
len /= 2;
for (i = self->len; --i >= 0; ) {
DECREF_KEY(self->keys[i]);
DECREF_VALUE(self->values[i]);
}
self->len = 0;
if (self->next) {
Py_DECREF(self->next);
self->next = NULL;
}
if (len > self->size) {
keys = BTree_Realloc(self->keys, sizeof(KEY_TYPE)*len);
if (keys == NULL)
return -1;
values = BTree_Realloc(self->values, sizeof(VALUE_TYPE)*len);
if (values == NULL)
return -1;
self->keys = keys;
self->values = values;
self->size = len;
}
for (i=0, l=0; i < len; i++) {
k = PyTuple_GET_ITEM(items, l);
l++;
v = PyTuple_GET_ITEM(items, l);
l++;
COPY_KEY_FROM_ARG(self->keys[i], k, copied);
if (!copied)
return -1;
COPY_VALUE_FROM_ARG(self->values[i], v, copied);
if (!copied)
return -1;
INCREF_KEY(self->keys[i]);
INCREF_VALUE(self->values[i]);
}
self->len = len;
if (next) {
self->next = next;
Py_INCREF(next);
}
return 0;
}
static PyObject *
bucket_setstate(Bucket *self, PyObject *state)
{
int r;
PER_PREVENT_DEACTIVATION(self);
r = _bucket_setstate(self, state);
PER_UNUSE(self);
if (r < 0)
return NULL;
Py_INCREF(Py_None);
return Py_None;
}
static PyObject *
bucket_has_key(Bucket *self, PyObject *key)
{
return _bucket_get(self, key, 1);
}
static PyObject *
bucket_setdefault(Bucket *self, PyObject *args)
{
PyObject *key;
PyObject *failobj; /* default */
PyObject *value; /* return value */
int dummy_changed; /* in order to call _bucket_set */
if (! PyArg_UnpackTuple(args, "setdefault", 2, 2, &key, &failobj))
return NULL;
value = _bucket_get(self, key, 0);
if (value != NULL)
return value;
/* The key isn't in the bucket. If that's not due to a KeyError exception,
* pass back the unexpected exception.
*/
if (! PyErr_ExceptionMatches(PyExc_KeyError))
return NULL;
PyErr_Clear();
/* Associate `key` with `failobj` in the bucket, and return `failobj`. */
value = failobj;
if (_bucket_set(self, key, failobj, 0, 0, &dummy_changed) < 0)
value = NULL;
Py_XINCREF(value);
return value;
}
/* forward declaration */
static int
Bucket_length(Bucket *self);
static PyObject *
bucket_pop(Bucket *self, PyObject *args)
{
PyObject *key;
PyObject *failobj = NULL; /* default */
PyObject *value; /* return value */
int dummy_changed; /* in order to call _bucket_set */
if (! PyArg_UnpackTuple(args, "pop", 1, 2, &key, &failobj))
return NULL;
value = _bucket_get(self, key, 0);
if (value != NULL) {
/* Delete key and associated value. */
if (_bucket_set(self, key, NULL, 0, 0, &dummy_changed) < 0) {
Py_DECREF(value);
return NULL;
}
return value;
}
/* The key isn't in the bucket. If that's not due to a KeyError exception,
* pass back the unexpected exception.
*/
if (! PyErr_ExceptionMatches(PyExc_KeyError))
return NULL;
if (failobj != NULL) {
/* Clear the KeyError and return the explicit default. */
PyErr_Clear();
Py_INCREF(failobj);
return failobj;
}
/* No default given. The only difference in this case is the error
* message, which depends on whether the bucket is empty.
*/
if (Bucket_length(self) == 0)
PyErr_SetString(PyExc_KeyError, "pop(): Bucket is empty");
return NULL;
}
/* Search bucket self for key. This is the sq_contains slot of the
* PySequenceMethods.
*
* Return:
* -1 error
* 0 not found
* 1 found
*/
static int
bucket_contains(Bucket *self, PyObject *key)
{
PyObject *asobj = _bucket_get(self, key, 1);
int result = -1;
if (asobj != NULL) {
result = PyInt_AsLong(asobj) ? 1 : 0;
Py_DECREF(asobj);
}
return result;
}
/*
** bucket_getm
**
*/
static PyObject *
bucket_getm(Bucket *self, PyObject *args)
{
PyObject *key, *d=Py_None, *r;
if (!PyArg_ParseTuple(args, "O|O:get", &key, &d))
return NULL;
r = _bucket_get(self, key, 0);
if (r)
return r;
if (!PyErr_ExceptionMatches(PyExc_KeyError))
return NULL;
PyErr_Clear();
Py_INCREF(d);
return d;
}
/**************************************************************************/
/* Iterator support. */
/* A helper to build all the iterators for Buckets and Sets.
* If args is NULL, the iterator spans the entire structure. Else it's an
* argument tuple, with optional low and high arguments.
* kind is 'k', 'v' or 'i'.
* Returns a BTreeIter object, or NULL if error.
*/
static PyObject *
buildBucketIter(Bucket *self, PyObject *args, PyObject *kw, char kind)
{
BTreeItems *items;
int lowoffset, highoffset;
BTreeIter *result = NULL;
PER_USE_OR_RETURN(self, NULL);
if (Bucket_rangeSearch(self, args, kw, &lowoffset, &highoffset) < 0)
goto Done;
items = (BTreeItems *)newBTreeItems(kind, self, lowoffset,
self, highoffset);
if (items == NULL) goto Done;
result = BTreeIter_new(items); /* win or lose, we're done */
Py_DECREF(items);
Done:
PER_UNUSE(self);
return (PyObject *)result;
}
/* The implementation of iter(Bucket_or_Set); the Bucket tp_iter slot. */
static PyObject *
Bucket_getiter(Bucket *self)
{
return buildBucketIter(self, NULL, NULL, 'k');
}
/* The implementation of Bucket.iterkeys(). */
static PyObject *
Bucket_iterkeys(Bucket *self, PyObject *args, PyObject *kw)
{
return buildBucketIter(self, args, kw, 'k');
}
/* The implementation of Bucket.itervalues(). */
static PyObject *
Bucket_itervalues(Bucket *self, PyObject *args, PyObject *kw)
{
return buildBucketIter(self, args, kw, 'v');
}
/* The implementation of Bucket.iteritems(). */
static PyObject *
Bucket_iteritems(Bucket *self, PyObject *args, PyObject *kw)
{
return buildBucketIter(self, args, kw, 'i');
}
/* End of iterator support. */
#ifdef PERSISTENT
static PyObject *merge_error(int p1, int p2, int p3, int reason);
static PyObject *bucket_merge(Bucket *s1, Bucket *s2, Bucket *s3);
static PyObject *
_bucket__p_resolveConflict(PyObject *ob_type, PyObject *s[3])
{
PyObject *result = NULL; /* guilty until proved innocent */
Bucket *b[3] = {NULL, NULL, NULL};
PyObject *meth = NULL;
PyObject *a = NULL;
int i;
for (i = 0; i < 3; i++) {
PyObject *r;
b[i] = (Bucket*)PyObject_CallObject((PyObject *)ob_type, NULL);
if (b[i] == NULL)
goto Done;
if (s[i] == Py_None) /* None is equivalent to empty, for BTrees */
continue;
meth = PyObject_GetAttr((PyObject *)b[i], __setstate___str);
if (meth == NULL)
goto Done;
a = PyTuple_New(1);
if (a == NULL)
goto Done;
PyTuple_SET_ITEM(a, 0, s[i]);
Py_INCREF(s[i]);
r = PyObject_CallObject(meth, a); /* b[i].__setstate__(s[i]) */
if (r == NULL)
goto Done;
Py_DECREF(r);
Py_DECREF(a);
Py_DECREF(meth);
a = meth = NULL;
}
if (b[0]->next != b[1]->next || b[0]->next != b[2]->next)
merge_error(-1, -1, -1, 0);
else
result = bucket_merge(b[0], b[1], b[2]);
Done:
Py_XDECREF(meth);
Py_XDECREF(a);
Py_XDECREF(b[0]);
Py_XDECREF(b[1]);
Py_XDECREF(b[2]);
return result;
}
static PyObject *
bucket__p_resolveConflict(Bucket *self, PyObject *args)
{
PyObject *s[3];
if (!PyArg_ParseTuple(args, "OOO", &s[0], &s[1], &s[2]))
return NULL;
return _bucket__p_resolveConflict((PyObject *)self->ob_type, s);
}
#endif
/* Caution: Even though the _next attribute is read-only, a program could
do arbitrary damage to the btree internals. For example, it could call
clear() on a bucket inside a BTree.
We need to decide if the convenience for inspecting BTrees is worth
the risk.
*/
static struct PyMemberDef Bucket_members[] = {
{"_next", T_OBJECT, offsetof(Bucket, next)},
{NULL}
};
static struct PyMethodDef Bucket_methods[] = {
{"__getstate__", (PyCFunction) bucket_getstate, METH_NOARGS,
"__getstate__() -- Return the picklable state of the object"},
{"__setstate__", (PyCFunction) bucket_setstate, METH_O,
"__setstate__() -- Set the state of the object"},
{"keys", (PyCFunction) bucket_keys, METH_KEYWORDS,
"keys([min, max]) -- Return the keys"},
{"has_key", (PyCFunction) bucket_has_key, METH_O,
"has_key(key) -- Test whether the bucket contains the given key"},
{"clear", (PyCFunction) bucket_clear, METH_VARARGS,
"clear() -- Remove all of the items from the bucket"},
{"update", (PyCFunction) Mapping_update, METH_O,
"update(collection) -- Add the items from the given collection"},
{"maxKey", (PyCFunction) Bucket_maxKey, METH_VARARGS,
"maxKey([key]) -- Find the maximum key\n\n"
"If an argument is given, find the maximum <= the argument"},
{"minKey", (PyCFunction) Bucket_minKey, METH_VARARGS,
"minKey([key]) -- Find the minimum key\n\n"
"If an argument is given, find the minimum >= the argument"},
{"values", (PyCFunction) bucket_values, METH_KEYWORDS,
"values([min, max]) -- Return the values"},
{"items", (PyCFunction) bucket_items, METH_KEYWORDS,
"items([min, max])) -- Return the items"},
{"byValue", (PyCFunction) bucket_byValue, METH_O,
"byValue(min) -- "
"Return value-keys with values >= min and reverse sorted by values"},
{"get", (PyCFunction) bucket_getm, METH_VARARGS,
"get(key[,default]) -- Look up a value\n\n"
"Return the default (or None) if the key is not found."},
{"setdefault", (PyCFunction) bucket_setdefault, METH_VARARGS,
"D.setdefault(k, d) -> D.get(k, d), also set D[k]=d if k not in D.\n\n"
"Return the value like get() except that if key is missing, d is both\n"
"returned and inserted into the bucket as the value of k."},
{"pop", (PyCFunction) bucket_pop, METH_VARARGS,
"D.pop(k[, d]) -> v, remove key and return the corresponding value.\n\n"
"If key is not found, d is returned if given, otherwise KeyError\n"
"is raised."},
{"iterkeys", (PyCFunction) Bucket_iterkeys, METH_KEYWORDS,
"B.iterkeys([min[,max]]) -> an iterator over the keys of B"},
{"itervalues", (PyCFunction) Bucket_itervalues, METH_KEYWORDS,
"B.itervalues([min[,max]]) -> an iterator over the values of B"},
{"iteritems", (PyCFunction) Bucket_iteritems, METH_KEYWORDS,
"B.iteritems([min[,max]]) -> an iterator over the (key, value) items of B"},
#ifdef EXTRA_BUCKET_METHODS
EXTRA_BUCKET_METHODS
#endif
#ifdef PERSISTENT
{"_p_resolveConflict", (PyCFunction) bucket__p_resolveConflict,
METH_VARARGS,
"_p_resolveConflict() -- Reinitialize from a newly created copy"},
{"_p_deactivate", (PyCFunction) bucket__p_deactivate, METH_KEYWORDS,
"_p_deactivate() -- Reinitialize from a newly created copy"},
#endif
{NULL, NULL}
};
static int
Bucket_init(PyObject *self, PyObject *args, PyObject *kwds)
{
PyObject *v = NULL;
if (!PyArg_ParseTuple(args, "|O:" MOD_NAME_PREFIX "Bucket", &v))
return -1;
if (v)
return update_from_seq(self, v);
else
return 0;
}
static void
bucket_dealloc(Bucket *self)
{
if (self->state != cPersistent_GHOST_STATE)
_bucket_clear(self);
cPersistenceCAPI->pertype->tp_dealloc((PyObject *)self);
}
static int
bucket_traverse(Bucket *self, visitproc visit, void *arg)
{
int err = 0;
int i, len;
#define VISIT(SLOT) \
if (SLOT) { \
err = visit((PyObject *)(SLOT), arg); \
if (err) \
goto Done; \
}
/* Call our base type's traverse function. Because buckets are
* subclasses of Peristent, there must be one.
*/
err = cPersistenceCAPI->pertype->tp_traverse((PyObject *)self, visit, arg);
if (err)
goto Done;
/* If this is registered with the persistence system, cleaning up cycles
* is the database's problem. It would be horrid to unghostify buckets
* here just to chase pointers every time gc runs.
*/
if (self->state == cPersistent_GHOST_STATE)
goto Done;
len = self->len;
(void)i; /* if neither keys nor values are PyObject*, "i" is otherwise
unreferenced and we get a nuisance compiler wng */
#ifdef KEY_TYPE_IS_PYOBJECT
/* Keys are Python objects so need to be traversed. */
for (i = 0; i < len; i++)
VISIT(self->keys[i]);
#endif
#ifdef VALUE_TYPE_IS_PYOBJECT
if (self->values != NULL) {
/* self->values exists (this is a mapping bucket, not a set bucket),
* and are Python objects, so need to be traversed. */
for (i = 0; i < len; i++)
VISIT(self->values[i]);
}
#endif
VISIT(self->next);
Done:
return err;
#undef VISIT
}
static int
bucket_tp_clear(Bucket *self)
{
if (self->state != cPersistent_GHOST_STATE)
_bucket_clear(self);
return 0;
}
/* Code to access Bucket objects as mappings */
static int
Bucket_length( Bucket *self)
{
int r;
UNLESS (PER_USE(self)) return -1;
r = self->len;
PER_UNUSE(self);
return r;
}
static PyMappingMethods Bucket_as_mapping = {
(lenfunc)Bucket_length, /*mp_length*/
(binaryfunc)bucket_getitem, /*mp_subscript*/
(objobjargproc)bucket_setitem, /*mp_ass_subscript*/
};
static PySequenceMethods Bucket_as_sequence = {
(lenfunc)0, /* sq_length */
(binaryfunc)0, /* sq_concat */
(ssizeargfunc)0, /* sq_repeat */
(ssizeargfunc)0, /* sq_item */
(ssizessizeargfunc)0, /* sq_slice */
(ssizeobjargproc)0, /* sq_ass_item */
(ssizessizeobjargproc)0, /* sq_ass_slice */
(objobjproc)bucket_contains, /* sq_contains */
0, /* sq_inplace_concat */
0, /* sq_inplace_repeat */
};
static PyObject *
bucket_repr(Bucket *self)
{
PyObject *i, *r;
char repr[10000];
int rv;
i = bucket_items(self, NULL, NULL);
if (!i)
return NULL;
r = PyObject_Repr(i);
Py_DECREF(i);
if (!r) {
return NULL;
}
rv = PyOS_snprintf(repr, sizeof(repr),
"%s(%s)", self->ob_type->tp_name,
PyString_AS_STRING(r));
if (rv > 0 && rv < sizeof(repr)) {
Py_DECREF(r);
return PyString_FromStringAndSize(repr, strlen(repr));
}
else {
/* The static buffer wasn't big enough */
int size;
PyObject *s;
/* 3 for the parens and the null byte */
size = strlen(self->ob_type->tp_name) + PyString_GET_SIZE(r) + 3;
s = PyString_FromStringAndSize(NULL, size);
if (!s) {
Py_DECREF(r);
return r;
}
PyOS_snprintf(PyString_AS_STRING(s), size,
"%s(%s)", self->ob_type->tp_name, PyString_AS_STRING(r));
Py_DECREF(r);
return s;
}
}
static PyTypeObject BucketType = {
PyObject_HEAD_INIT(NULL) /* PyPersist_Type */
0, /* ob_size */
MODULE_NAME MOD_NAME_PREFIX "Bucket",/* tp_name */
sizeof(Bucket), /* tp_basicsize */
0, /* tp_itemsize */
(destructor)bucket_dealloc, /* tp_dealloc */
0, /* tp_print */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_compare */
(reprfunc)bucket_repr, /* tp_repr */
0, /* tp_as_number */
&Bucket_as_sequence, /* tp_as_sequence */
&Bucket_as_mapping, /* tp_as_mapping */
0, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
0, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
Py_TPFLAGS_BASETYPE, /* tp_flags */
0, /* tp_doc */
(traverseproc)bucket_traverse, /* tp_traverse */
(inquiry)bucket_tp_clear, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
(getiterfunc)Bucket_getiter, /* tp_iter */
0, /* tp_iternext */
Bucket_methods, /* tp_methods */
Bucket_members, /* tp_members */
0, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
Bucket_init, /* tp_init */
0, /* tp_alloc */
0, /*PyType_GenericNew,*/ /* tp_new */
};
static int
nextBucket(SetIteration *i)
{
if (i->position >= 0)
{
UNLESS(PER_USE(BUCKET(i->set))) return -1;
if (i->position)
{
DECREF_KEY(i->key);
DECREF_VALUE(i->value);
}
if (i->position < BUCKET(i->set)->len)
{
COPY_KEY(i->key, BUCKET(i->set)->keys[i->position]);
INCREF_KEY(i->key);
COPY_VALUE(i->value, BUCKET(i->set)->values[i->position]);
INCREF_VALUE(i->value);
i->position ++;
}
else
{
i->position = -1;
PER_ACCESSED(BUCKET(i->set));
}
PER_ALLOW_DEACTIVATION(BUCKET(i->set));
}
return 0;
}
BTrees/Development.txt 0 → 100644
View file @ 703fc7e7
=====================
Developer Information
=====================
This document provides information for developers who maintain or extend
`BTrees`.
Macros
======
`BTrees` are defined using a "template", roughly akin to a C++ template. To
create a new family of `BTrees`, create a source file that defines macros used
to handle differences in key and value types:
Configuration Macros
--------------------
``MASTER_ID``
A string to hold an RCS/CVS Id key to be included in compiled binaries.
``MOD_NAME_PREFIX``
A string (like "IO" or "OO") that provides the prefix used for the module.
This gets used to generate type names and the internal module name string.
``DEFAULT_MAX_BUCKET_SIZE``
An int giving the maximum bucket size (number of key/value pairs). When a
bucket gets larger than this due to an insertion *into a BTREE*, it
splits. Inserting into a bucket directly doesn't split, and functions
that produce a bucket output (e.g., ``union()``) also have no bound on how
large a bucket may get. Someday this will be tunable on `BTree`.
instances.
``DEFAULT_MAX_BTREE_SIZE``
An ``int`` giving the maximum size (number of children) of an internal
btree node. Someday this will be tunable on ``BTree`` instances.
Macros for Keys
---------------
``KEY_TYPE``
The C type declaration for keys (e.g., ``int`` or ``PyObject*``).
``KEY_TYPE_IS_PYOBJECT``
Define if ``KEY_TYPE`` is a ``PyObject*`, else ``undef``.
``KEY_CHECK(K)``
Tests whether the ``PyObject* K`` can be converted to the (``C``) key type
(``KEY_TYPE``). The macro should return a boolean (zero for false,
non-zero for true). When it returns false, its caller should probably set
a ``TypeError`` exception.
``TEST_KEY_SET_OR(V, K, T)``
Like Python's ``cmp()``. Compares K(ey) to T(arget), where ``K``
and ``T`` are ``C`` values of type `KEY_TYPE`. ``V`` is assigned an `int`
value depending on the outcome::
< 0 if K < T
== 0 if K == T
> 0 if K > T
This macro acts like an ``if``, where the following statement is executed
only if a Python exception has been raised because the values could not be
compared.
``DECREF_KEY(K)``
``K`` is a value of ``KEY_TYPE``. If ``KEY_TYPE`` is a flavor of
``PyObject*``, write this to do ``Py_DECREF(K)``. Else (e.g.,
``KEY_TYPE`` is ``int``) make it a nop.
``INCREF_KEY(K)``
``K`` is a value of `KEY_TYPE`. If `KEY_TYPE` is a flavor of
``PyObject*``, write this to do ``Py_INCREF(K)``. Else (e.g., `KEY_TYPE`
is ``int``) make it a nop.
``COPY_KEY(K, E)``
Like ``K=E``. Copy a key from ``E`` to ``K``, both of ``KEY_TYPE``. Note
that this doesn't ``decref K`` or ``incref E`` when ``KEY_TYPE`` is a
``PyObject*``; the caller is responsible for keeping refcounts straight.
``COPY_KEY_TO_OBJECT(O, K)``
Roughly like ``O=K``. ``O`` is a ``PyObject*``, and the macro must build
a Python object form of ``K``, assign it to ``O``, and ensure that ``O``
owns the reference to its new value. It may do this by creating a new
Python object based on ``K`` (e.g., ``PyInt_FromLong(K)`` when
``KEY_TYPE`` is ``int``), or simply by doing ``Py_INCREF(K)`` if
``KEY_TYPE`` is a ``PyObject*``.
``COPY_KEY_FROM_ARG(TARGET, ARG, STATUS)``
Copy an argument to the target without creating a new reference to
``ARG``. ``ARG`` is a ``PyObject*``, and ``TARGET`` is of type
``KEY_TYPE``. If this can't be done (for example, ``KEY_CHECK(ARG)``
returns false), set a Python error and set status to ``0``. If there is
no error, leave status alone.
Macros for Values
-----------------
``VALUE_TYPE``
The C type declaration for values (e.g., ``int`` or ``PyObject*``).
``VALUE_TYPE_IS_PYOBJECT``
Define if ``VALUE_TYPE`` is a ``PyObject*``, else ``undef``.
``TEST_VALUE(X, Y)``
Like Python's ``cmp()``. Compares ``X`` to ``Y``, where ``X`` & ``Y`` are
``C`` values of type ``VALUE_TYPE``. The macro returns an ``int``, with
value::
< 0 if X < Y
== 0 if X == Y
> 0 if X > Y
Bug: There is no provision for determining whether the comparison attempt
failed (set a Python exception).
``DECREF_VALUE(K)``
Like ``DECREF_KEY``, except applied to values of ``VALUE_TYPE``.
``INCREF_VALUE(K)``
Like ``INCREF_KEY``, except applied to values of ``VALUE_TYPE``.
``COPY_VALUE(K, E)``
Like ``COPY_KEY``, except applied to values of ``VALUE_TYPE``.
``COPY_VALUE_TO_OBJECT(O, K)``
Like ``COPY_KEY_TO_OBJECT``, except applied to values of ``VALUE_TYPE``.
``COPY_VALUE_FROM_ARG(TARGET, ARG, STATUS)``
Like ``COPY_KEY_FROM_ARG``, except applied to values of ``VALUE_TYPE``.
``NORMALIZE_VALUE(V, MIN)``
Normalize the value, ``V``, using the parameter ``MIN``. This is almost
certainly a YAGNI. It is a no-op for most types. For integers, ``V`` is
replaced by ``V/MIN`` only if ``MIN > 0``.
Macros for Set Operations
-------------------------
``MERGE_DEFAULT``
A value of ``VALUE_TYPE`` specifying the value to associate with set
elements when sets are merged with mappings via weighed union or weighted
intersection.
``MERGE(O1, w1, O2, w2)``
Performs a weighted merge of two values, ``O1`` and ``O2``, using weights
``w1`` and ``w2``. The result must be of ``VALUE_TYPE``. Note that
weighted unions and weighted intersections are not enabled if this macro
is left undefined.
``MERGE_WEIGHT(O, w)``
Computes a weighted value for ``O``. The result must be of
``VALUE_TYPE``. This is used for "filling out" weighted unions, i.e. to
compute a weighted value for keys that appear in only one of the input
mappings. If left undefined, ``MERGE_WEIGHT`` defaults to::
#define MERGE_WEIGHT(O, w) (O)
``MULTI_INT_UNION``
The value doesn't matter. If defined, `SetOpTemplate.c` compiles code for
a ``multiunion()`` function (compute a union of many input sets at high
speed). This currently makes sense only for structures with integer keys.
BTree Clues
===========
More or less random bits of helpful info.
+ In papers and textbooks, this flavor of BTree is usually called a B+-Tree,
where "+" is a superscript.
+ All keys and all values live in the bucket leaf nodes. Keys in interior
(BTree) nodes merely serve to guide a search efficiently toward the correct
leaf.
+ When a key is deleted, it's physically removed from the bucket it's in, but
this doesn't propagate back up the tree: since keys in interior nodes only
serve to guide searches, it's OK-- and saves time --to leave "stale" keys in
interior nodes.
+ No attempt is made to rebalance the tree after a deletion, unless a bucket
thereby becomes entirely empty. "Classic BTrees" do rebalance, keeping all
buckets at least half full (provided there are enough keys in the entire
tree to fill half a bucket). The tradeoffs are murky. Pathological cases
in the presence of deletion do exist. Pathologies include trees tending
toward only one key per bucket, and buckets at differing depths (all buckets
are at the same depth in a classic BTree).
+ ``DEFAULT_MAX_BUCKET_SIZE`` and ``DEFAULT_MAX_BTREE_SIZE`` are chosen mostly
to "even out" pickle sizes in storage. That's why, e.g., an `IIBTree` has
larger values than an `OOBTree`: pickles store ints more efficiently than
they can store arbitrary Python objects.
+ In a non-empty BTree, every bucket node contains at least one key, and every
BTree node contains at least one child and a non-NULL firstbucket pointer.
However, a BTree node may not contain any keys.
+ An empty BTree consists solely of a BTree node with ``len==0`` and
``firstbucket==NULL``.
+ Although a BTree can become unbalanced under a mix of inserts and deletes
(meaning both that there's nothing stronger that can be said about buckets
than that they're not empty, and that buckets can appear at different
depths), a BTree node always has children of the same kind: they're all
buckets, or they're all BTree nodes.
The ``BTREE_SEARCH`` Macro
==========================
For notational ease, consider a fixed BTree node ``x``, and let
::
K(i) mean x->data.key[i]
C(i) mean all the keys reachable from x->data.child[i]
For each ``i`` in ``0`` to ``x->len-1`` inclusive,
::
K(i) <= C(i) < K(i+1)
is a BTree node invariant, where we pretend that ``K(0)`` holds a key smaller
than any possible key, and ``K(x->len)`` holds a key larger than any possible
key. (Note that ``K(x->len)`` doesn't actually exist, and ``K(0)`` is never
used although space for it exists in non-empty BTree nodes.)
When searching for a key ``k``, then, the child pointer we want to follow is
the one at index ``i`` such that ``K(i) <= k < K(i+1)``. There can be at most
one such ``i``, since the ``K(i)`` are strictly increasing. And there is at
least one such ``i`` provided the tree isn't empty (so that ``0 < len``). For
the moment, assume the tree isn't empty (we'll get back to that later).
The macro's chief loop invariant is
::
K(lo) < k < K(hi)
This holds trivially at the start, since ``lo`` is set to ``0``, and ``hi`` to
``x->len``, and we pretend ``K(0)`` is minus infinity and ``K(len)`` is plus
infinity. Inside the loop, if ``K(i) < k`` we set ``lo`` to ``i``, and if
``K(i) > k`` we set ``hi`` to ``i``. These obviously preserve the invariant.
If ``K(i) == k``, the loop breaks and sets the result to ``i``, and since
``K(i) == k`` in that case ``i`` is obviously the correct result.
Other cases depend on how ``i = floor((lo + hi)/2)`` works, exactly. Suppose
``lo + d = hi`` for some ``d >= 0``. Then ``i = floor((lo + lo + d)/2) =
floor(lo + d/2) = lo + floor(d/2)``. So:
a. ``[d == 0] (lo == i == hi)`` if and only if ``(lo == hi)``.
b. ``[d == 1] (lo == i < hi)`` if and only if ``(lo+1 == hi)``.
c. ``[d > 1] (lo < i < hi)`` if and only if ``(lo+1 < hi)``.
If the node is empty ``(x->len == 0)``, then ``lo==i==hi==0`` at the start,
and the loop exits immediately (the first ``i > lo`` test fails), without
entering the body.
Else ``lo < hi`` at the start, and the invariant ``K(lo) < k < K(hi)`` holds.
If ``lo+1 < hi``, we're in case (c): ``i`` is strictly between ``lo`` and
``hi``, so the loop body is entered, and regardless of whether the body sets
the new ``lo`` or the new ``hi`` to ``i``, the new ``lo`` is strictly less
than the new ``hi``, and the difference between the new ``lo`` and new ``hi``
is strictly less than the difference between the old ``lo`` and old ``hi``.
So long as the new ``lo + 1`` remains < the new ``hi``, we stay in this case.
We can't stay in this case forever, though: because ``hi-lo`` decreases on
each trip but remains > ``0``, ``lo+1 == hi`` must eventually become true.
(In fact, it becomes true quickly, in about ``log2(x->len)`` trips; the point
is more that ``lo`` doesn't equal ``hi`` when the loop ends, it has to end
with ``lo+1==hi`` and ``i==lo``).
Then we're in case (b): ``i==lo==hi-1`` then, and the loop exits. The
invariant still holds, with ``lo==i`` and ``hi==lo+1==i+1``::
K(i) < k < K(i+1)
so ``i`` is again the correct answer.
Optimization points:
--------------------
+ Division by 2 is done via shift rather via "/2". These are signed ints, and
almost all C compilers treat signed int division as truncating, and shifting
is not the same as truncation for signed int division. The compiler has no
way to know these values aren't negative, so has to generate longer-winded
code for "/2". But we know these values aren't negative, and exploit it.
+ The order of _cmp comparisons matters. We're in an interior BTree node, and
are looking at only a tiny fraction of all the keys that exist. So finding
the key exactly in this node is unlikely, and checking ``_cmp == 0`` is a
waste of time to the same extent. It doesn't matter whether we check for
``_cmp < 0`` or ``_cmp > 0`` first, so long as we do both before worrying
about equality.
+ At the start of a routine, it's better to run this macro even if ``x->len``
is ``0`` (check for that afterwards). We just called a function and so
probably drained the pipeline. If the first thing we do then is read up
``self->len`` and check it against ``0``, we just sit there waiting for the
data to get read up, and then another immediate test-and-branch, and for a
very unlikely case (BTree nodes are rarely empty). It's better to get into
the loop right away so the normal case makes progress ASAP.
The ``BUCKET_SEARCH`` Macro
===========================
This has a different job than ``BTREE_SEARCH``: the key ``0`` slot is
legitimate in a bucket, and we want to find the index at which the key
belongs. If the key is larger than the bucket's largest key, a new slot at
index len is where it belongs, else it belongs at the smallest ``i`` with
``keys[i]`` >= the key we're looking for. We also need to know whether or not
the key is present (``BTREE_SEARCH`` didn't care; it only wanted to find the
next node to search).
The mechanics of the search are quite similar, though. The primary
loop invariant changes to (say we're searching for key ``k``)::
K(lo-1) < k < K(hi)
where ``K(i)`` means ``keys[i]``, and we pretend ``K(-1)`` is minus infinity
and ``K(len)`` is plus infinity.
If the bucket is empty, ``lo=hi=i=0`` at the start, the loop body is never
entered, and the macro sets ``INDEX`` to 0 and ``ABSENT`` to true. That's why
``_cmp`` is initialized to 1 (``_cmp`` becomes ``ABSENT``).
Else the bucket is not empty, lo<hi at the start, and the loop body is
entered. The invariant is obviously satisfied then, as ``lo=0`` and
``hi=len``.
If ``K[i]<k``, ``lo`` is set to ``i+1``, preserving that ``K(lo-1) = K[i] <
k``.
If ``K[i]>k``, ``hi`` is set to ``i``, preserving that ``K[hi] = K[i] > k``.
If the loop exits after either of those, ``_cmp != 0``, so ``ABSENT`` becomes
true.
If ``K[i]=k``, the loop breaks, so that ``INDEX`` becomes ``i``, and
``ABSENT`` becomes false (``_cmp=0`` in this case).
The same case analysis for ``BTREE_SEARCH`` on ``lo`` and ``hi`` holds here:
a. ``(lo == i == hi)`` if and only if ``(lo == hi)``.
b. ``(lo == i < hi)`` if and only if ``(lo+1 == hi)``.
c. ``(lo < i < hi)`` if and only if ``(lo+1 < hi)``.
So long as ``lo+1 < hi``, we're in case (c), and either break with equality
(in which case the right results are obviously computed) or narrow the range.
If equality doesn't obtain, the range eventually narrows to cases (a) or (b).
To go from (c) to (a), we must have ``lo+2==hi`` at the start, and
``K[i]=K[lo+1]<k``. Then the new lo gets set to ``i+1 = lo+2 = hi``, and the
loop exits with ``lo=hi=i`` and ``_cmp<0``. This is correct, because we know
that ``k != K(i)`` (loop invariant! we actually know something stronger, that
``k < K(hi)``; since ``i=hi``, this implies ``k != K(i)``).
Else (c) eventually falls into case (b), ``lo+1==hi`` and ``i==lo``. The
invariant tells us ``K(lo-1) < k < K(hi) = K(lo+1)``, so if the key is present
it must be at ``K(lo)``. ``i==lo`` in this case, so we test ``K(lo)`` against
``k``. As always, if equality obtains we do the right thing, else case #b
becomes case (a).
When (b) becomes (a), the last comparison was non-equal, so ``_cmp`` is
non-zero, and the loop exits because ``lo==hi==i`` in case (a). The invariant
then tells us ``K(lo-1) < k < K(lo)``, so the key is in fact not present, it's
correct to exit with ``_cmp`` non-zero, and ``i==lo`` is again the index at
which ``k`` belongs.
Optimization points:
--------------------
+ As for ``BTREE_SEARCH``, shifting of signed ints is cheaper than division.
+ Unlike as for ``BTREE_SEARCH``, there's nothing special about searching an
empty bucket, and the macro computes thoroughly sensible results in that
case.
+ The order of ``_cmp`` comparisons differs from ``BTREE_SEARCH``. When
searching a bucket, it's much more likely (than when searching a BTree node)
that the key is present, so testing ``__cmp==0`` isn't a systematic waste of
cycles. At the extreme, if all searches are successful (key present), on
average this saves one comparison per search, against leaving the
determination of ``_cmp==0`` implicit (as ``BTREE_SEARCH`` does). But even
on successful searches, ``__cmp != 0`` is a more popular outcome than
``__cmp == 0`` across iterations (unless the bucket has only a few keys), so
it's important to check one of the inequality cases first. It turns out
it's better on average to check ``K(i) < key`` (than to check ``K(i) >
key``), because when it pays it narrows the range more (we get a little
boost from setting ``lo=i+1`` in this case; the other case sets ``hi=i``,
which isn't as much of a narrowing).
BTrees/IFBTree.py 0 → 100644
View file @ 703fc7e7
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import zope.interface
import BTrees.Interfaces
# hack to overcome dynamic-linking headache.
from _IFBTree import *
zope.interface.moduleProvides(BTrees.Interfaces.IIntegerFloatBTreeModule)
BTrees/IIBTree.py 0 → 100644
View file @ 703fc7e7
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import zope.interface
import BTrees.Interfaces
# hack to overcome dynamic-linking headache.
from _IIBTree import *
zope.interface.moduleProvides(BTrees.Interfaces.IIntegerIntegerBTreeModule)
BTrees/IOBTree.py 0 → 100644
View file @ 703fc7e7
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import zope.interface
import BTrees.Interfaces
# hack to overcome dynamic-linking headache.
from _IOBTree import *
zope.interface.moduleProvides(BTrees.Interfaces.IIntegerObjectBTreeModule)
BTrees/Interfaces.py 0 → 100644
View file @ 703fc7e7
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
from zope.interface import Interface, Attribute
class ICollection(Interface):
def clear():
"""Remove all of the items from the collection."""
def __nonzero__():
"""Check if the collection is non-empty.
Return a true value if the collection is non-empty and a
false value otherwise.
"""
class IReadSequence(Interface):
def __getitem__(index):
"""Return the value at the given index.
An IndexError is raised if the index cannot be found.
"""
def __getslice__(index1, index2):
"""Return a subsequence from the original sequence.
The subsequence includes the items from index1 up to, but not
including, index2.
"""
class IKeyed(ICollection):
def has_key(key):
"""Check whether the object has an item with the given key.
Return a true value if the key is present, else a false value.
"""
def keys(min=None, max=None, excludemin=False, excludemax=False):
"""Return an IReadSequence containing the keys in the collection.
The type of the IReadSequence is not specified. It could be a list
or a tuple or some other type.
All arguments are optional, and may be specified as keyword
arguments, or by position.
If a min is specified, then output is constrained to keys greater
than or equal to the given min, and, if excludemin is specified and
true, is further constrained to keys strictly greater than min. A
min value of None is ignored. If min is None or not specified, and
excludemin is true, the smallest key is excluded.
If a max is specified, then output is constrained to keys less than
or equal to the given max, and, if excludemax is specified and
true, is further constrained to keys strictly less than max. A max
value of None is ignored. If max is None or not specified, and
excludemax is true, the largest key is excluded.
"""
def maxKey(key=None):
"""Return the maximum key.
If a key argument if provided and not None, return the largest key
that is less than or equal to the argument. Raise an exception if
no such key exists.
"""
def minKey(key=None):
"""Return the minimum key.
If a key argument if provided and not None, return the smallest key
that is greater than or equal to the argument. Raise an exception
if no such key exists.
"""
class ISetMutable(IKeyed):
def insert(key):
"""Add the key (value) to the set.
If the key was already in the set, return 0, otherwise return 1.
"""
def remove(key):
"""Remove the key from the set.
Raises KeyError if key is not in the set.
"""
def update(seq):
"""Add the items from the given sequence to the set."""
class ISized(Interface):
"""An object that supports __len__."""
def __len__():
"""Return the number of items in the container."""
class IKeySequence(IKeyed, ISized):
def __getitem__(index):
"""Return the key in the given index position.
This allows iteration with for loops and use in functions,
like map and list, that read sequences.
"""
class ISet(IKeySequence, ISetMutable):
pass
class ITreeSet(IKeyed, ISetMutable):
pass
class IMinimalDictionary(ISized, IKeyed):
def get(key, default):
"""Get the value associated with the given key.
Return the default if has_key(key) is false.
"""
def __getitem__(key):
"""Get the value associated with the given key.
Raise KeyError if has_key(key) is false.
"""
def __setitem__(key, value):
"""Set the value associated with the given key."""
def __delitem__(key):
"""Delete the value associated with the given key.
Raise KeyError if has_key(key) is false.
"""
def values(min=None, max=None, excludemin=False, excludemax=False):
"""Return an IReadSequence containing the values in the collection.
The type of the IReadSequence is not specified. It could be a list
or a tuple or some other type.
All arguments are optional, and may be specified as keyword
arguments, or by position.
If a min is specified, then output is constrained to values whose
keys are greater than or equal to the given min, and, if excludemin
is specified and true, is further constrained to values whose keys
are strictly greater than min. A min value of None is ignored. If
min is None or not specified, and excludemin is true, the value
corresponding to the smallest key is excluded.
If a max is specified, then output is constrained to values whose
keys are less than or equal to the given max, and, if excludemax is
specified and true, is further constrained to values whose keys are
strictly less than max. A max value of None is ignored. If max is
None or not specified, and excludemax is true, the value
corresponding to the largest key is excluded.
"""
def items(min=None, max=None, excludemin=False, excludemax=False):
"""Return an IReadSequence containing the items in the collection.
An item is a 2-tuple, a (key, value) pair.
The type of the IReadSequence is not specified. It could be a list
or a tuple or some other type.
All arguments are optional, and may be specified as keyword
arguments, or by position.
If a min is specified, then output is constrained to items whose
keys are greater than or equal to the given min, and, if excludemin
is specified and true, is further constrained to items whose keys
are strictly greater than min. A min value of None is ignored. If
min is None or not specified, and excludemin is true, the item with
the smallest key is excluded.
If a max is specified, then output is constrained to items whose
keys are less than or equal to the given max, and, if excludemax is
specified and true, is further constrained to items whose keys are
strictly less than max. A max value of None is ignored. If max is
None or not specified, and excludemax is true, the item with the
largest key is excluded.
"""
class IDictionaryIsh(IMinimalDictionary):
def update(collection):
"""Add the items from the given collection object to the collection.
The input collection must be a sequence of (key, value) 2-tuples,
or an object with an 'items' method that returns a sequence of
(key, value) pairs.
"""
def byValue(minValue):
"""Return a sequence of (value, key) pairs, sorted by value.
Values < minValue are omitted and other values are "normalized" by
the minimum value. This normalization may be a noop, but, for
integer values, the normalization is division.
"""
def setdefault(key, d):
"""D.setdefault(k, d) -> D.get(k, d), also set D[k]=d if k not in D.
Return the value like get() except that if key is missing, d is both
returned and inserted into the dictionary as the value of k.
Note that, unlike as for Python's dict.setdefault(), d is not
optional. Python defaults d to None, but that doesn't make sense
for mappings that can't have None as a value (for example, an
IIBTree can have only integers as values).
"""
def pop(key, d):
"""D.pop(k[, d]) -> v, remove key and return the corresponding value.
If key is not found, d is returned if given, otherwise KeyError is
raised.
"""
class IBTree(IDictionaryIsh):
def insert(key, value):
"""Insert a key and value into the collection.
If the key was already in the collection, then there is no
change and 0 is returned.
If the key was not already in the collection, then the item is
added and 1 is returned.
This method is here to allow one to generate random keys and
to insert and test whether the key was there in one operation.
A standard idiom for generating new keys will be::
key = generate_key()
while not t.insert(key, value):
key=generate_key()
"""
class IMerge(Interface):
"""Object with methods for merging sets, buckets, and trees.
These methods are supplied in modules that define collection
classes with particular key and value types. The operations apply
only to collections from the same module. For example, the
IIBTree.union can only be used with IIBTree.IIBTree,
IIBTree.IIBucket, IIBTree.IISet, and IIBTree.IITreeSet.
The implementing module has a value type. The IOBTree and OOBTree
modules have object value type. The IIBTree and OIBTree modules
have integer value types. Other modules may be defined in the
future that have other value types.
The individual types are classified into set (Set and TreeSet) and
mapping (Bucket and BTree) types.
"""
def difference(c1, c2):
"""Return the keys or items in c1 for which there is no key in c2.
If c1 is None, then None is returned. If c2 is None, then c1
is returned.
If neither c1 nor c2 is None, the output is a Set if c1 is a Set or
TreeSet, and is a Bucket if c1 is a Bucket or BTree.
"""
def union(c1, c2):
"""Compute the Union of c1 and c2.
If c1 is None, then c2 is returned, otherwise, if c2 is None,
then c1 is returned.
The output is a Set containing keys from the input
collections.
"""
def intersection(c1, c2):
"""Compute the intersection of c1 and c2.
If c1 is None, then c2 is returned, otherwise, if c2 is None,
then c1 is returned.
The output is a Set containing matching keys from the input
collections.
"""
class IBTreeModule(Interface):
"""These are available in all modules (IOBTree, OIBTree, OOBTree, IIBTree,
IFBTree, LFBTree, LOBTree, OLBTree, and LLBTree).
"""
BTree = Attribute(
"""The IBTree for this module.
Also available as [prefix]BTree, as in IOBTree.""")
Bucket = Attribute(
"""The leaf-node data buckets used by the BTree.
(IBucket is not currently defined in this file, but is essentially
IDictionaryIsh, with the exception of __nonzero__, as of this
writing.)
Also available as [prefix]Bucket, as in IOBucket.""")
TreeSet = Attribute(
"""The ITreeSet for this module.
Also available as [prefix]TreeSet, as in IOTreeSet.""")
Set = Attribute(
"""The ISet for this module: the leaf-node data buckets used by the
TreeSet.
Also available as [prefix]BTree, as in IOSet.""")
class IIMerge(IMerge):
"""Merge collections with integer value type.
A primary intent is to support operations with no or integer
values, which are used as "scores" to rate indiviual keys. That
is, in this context, a BTree or Bucket is viewed as a set with
scored keys, using integer scores.
"""
def weightedUnion(c1, c2, weight1=1, weight2=1):
"""Compute the weighted union of c1 and c2.
If c1 and c2 are None, the output is (0, None).
If c1 is None and c2 is not None, the output is (weight2, c2).
If c1 is not None and c2 is None, the output is (weight1, c1).
Else, and hereafter, c1 is not None and c2 is not None.
If c1 and c2 are both sets, the output is 1 and the (unweighted)
union of the sets.
Else the output is 1 and a Bucket whose keys are the union of c1 and
c2's keys, and whose values are::
v1*weight1 + v2*weight2
where:
v1 is 0 if the key is not in c1
1 if the key is in c1 and c1 is a set
c1[key] if the key is in c1 and c1 is a mapping
v2 is 0 if the key is not in c2
1 if the key is in c2 and c2 is a set
c2[key] if the key is in c2 and c2 is a mapping
Note that c1 and c2 must be collections.
"""
def weightedIntersection(c1, c2, weight1=1, weight2=1):
"""Compute the weighted intersection of c1 and c2.
If c1 and c2 are None, the output is (0, None).
If c1 is None and c2 is not None, the output is (weight2, c2).
If c1 is not None and c2 is None, the output is (weight1, c1).
Else, and hereafter, c1 is not None and c2 is not None.
If c1 and c2 are both sets, the output is the sum of the weights
and the (unweighted) intersection of the sets.
Else the output is 1 and a Bucket whose keys are the intersection of
c1 and c2's keys, and whose values are::
v1*weight1 + v2*weight2
where:
v1 is 1 if c1 is a set
c1[key] if c1 is a mapping
v2 is 1 if c2 is a set
c2[key] if c2 is a mapping
Note that c1 and c2 must be collections.
"""
class IMergeIntegerKey(IMerge):
"""IMerge-able objects with integer keys.
Concretely, this means the types in IOBTree and IIBTree.
"""
def multiunion(seq):
"""Return union of (zero or more) integer sets, as an integer set.
seq is a sequence of objects each convertible to an integer set.
These objects are convertible to an integer set:
+ An integer, which is added to the union.
+ A Set or TreeSet from the same module (for example, an
IIBTree.TreeSet for IIBTree.multiunion()). The elements of the
set are added to the union.
+ A Bucket or BTree from the same module (for example, an
IOBTree.IOBTree for IOBTree.multiunion()). The keys of the
mapping are added to the union.
The union is returned as a Set from the same module (for example,
IIBTree.multiunion() returns an IIBTree.IISet).
The point to this method is that it can run much faster than
doing a sequence of two-input union() calls. Under the covers,
all the integers in all the inputs are sorted via a single
linear-time radix sort, then duplicates are removed in a second
linear-time pass.
"""
class IBTreeFamily(Interface):
"""the 64-bit or 32-bit family"""
IO = Attribute('The IIntegerObjectBTreeModule for this family')
OI = Attribute('The IObjectIntegerBTreeModule for this family')
II = Attribute('The IIntegerIntegerBTreeModule for this family')
IF = Attribute('The IIntegerFloatBTreeModule for this family')
OO = Attribute('The IObjectObjectBTreeModule for this family')
maxint = Attribute('The maximum integer storable in this family')
minint = Attribute('The minimum integer storable in this family')
class IIntegerObjectBTreeModule(IBTreeModule, IMerge):
"""keys, or set values, are integers; values are objects.
describes IOBTree and LOBTree"""
family = Attribute('The IBTreeFamily of this module')
class IObjectIntegerBTreeModule(IBTreeModule, IIMerge):
"""keys, or set values, are objects; values are integers.
Object keys (and set values) must sort reliably (for instance, *not* on
object id)! Homogenous key types recommended.
describes OIBTree and LOBTree"""
family = Attribute('The IBTreeFamily of this module')
class IIntegerIntegerBTreeModule(IBTreeModule, IIMerge, IMergeIntegerKey):
"""keys, or set values, are integers; values are also integers.
describes IIBTree and LLBTree"""
family = Attribute('The IBTreeFamily of this module')
class IObjectObjectBTreeModule(IBTreeModule, IMerge):
"""keys, or set values, are objects; values are also objects.
Object keys (and set values) must sort reliably (for instance, *not* on
object id)! Homogenous key types recommended.
describes OOBTree"""
# Note that there's no ``family`` attribute; all families include
# the OO flavor of BTrees.
class IIntegerFloatBTreeModule(IBTreeModule, IMerge):
"""keys, or set values, are integers; values are floats.
describes IFBTree and LFBTree"""
family = Attribute('The IBTreeFamily of this module')
###############################################################
# IMPORTANT NOTE
#
# Getting the length of a BTree, TreeSet, or output of keys,
# values, or items of same is expensive. If you need to get the
# length, you need to maintain this separately.
#
# Eventually, I need to express this through the interfaces.
#
################################################################
BTrees/LFBTree.py 0 → 100644
View file @ 703fc7e7
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import zope.interface
import BTrees.Interfaces
# hack to overcome dynamic-linking headache.
from _LFBTree import *
zope.interface.moduleProvides(BTrees.Interfaces.IIntegerFloatBTreeModule)
BTrees/LLBTree.py 0 → 100644
View file @ 703fc7e7
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import zope.interface
import BTrees.Interfaces
# hack to overcome dynamic-linking headache.
from _LLBTree import *
zope.interface.moduleProvides(BTrees.Interfaces.IIntegerIntegerBTreeModule)
BTrees/LOBTree.py 0 → 100644
View file @ 703fc7e7
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import zope.interface
import BTrees.Interfaces
# hack to overcome dynamic-linking headache.
from _LOBTree import *
zope.interface.moduleProvides(BTrees.Interfaces.IIntegerObjectBTreeModule)
BTrees/Length.py 0 → 100644
View file @ 703fc7e7
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import persistent
class Length(persistent.Persistent):
"""BTree lengths are often too expensive to compute.
Objects that use BTrees need to keep track of lengths themselves.
This class provides an object for doing this.
As a bonus, the object support application-level conflict
resolution.
It is tempting to to assign length objects to __len__ attributes
to provide instance-specific __len__ methods. However, this no
longer works as expected, because new-style classes cache
class-defined slot methods (like __len__) in C type slots. Thus,
instance-defined slot fillers are ignored.
"""
value = 0
def __init__(self, v=0):
self.value = v
def __getstate__(self):
return self.value
def __setstate__(self, v):
self.value = v
def set(self, v):
self.value = v
def _p_resolveConflict(self, old, s1, s2):
return s1 + s2 - old
def change(self, delta):
self.value += delta
def __call__(self, *args):
return self.value
BTrees/MergeTemplate.c 0 → 100644
View file @ 703fc7e7
/*****************************************************************************
Copyright (c) 2001, 2002 Zope Foundation and Contributors.
All Rights Reserved.
This software is subject to the provisions of the Zope Public License,
Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
****************************************************************************/
#define MERGETEMPLATE_C "$Id$\n"
/****************************************************************************
Set operations
****************************************************************************/
static int
merge_output(Bucket *r, SetIteration *i, int mapping)
{
if (r->len >= r->size && Bucket_grow(r, -1, !mapping) < 0)
return -1;
COPY_KEY(r->keys[r->len], i->key);
INCREF_KEY(r->keys[r->len]);
if (mapping) {
COPY_VALUE(r->values[r->len], i->value);
INCREF_VALUE(r->values[r->len]);
}
r->len++;
return 0;
}
/* The "reason" argument is a little integer giving "a reason" for the
* error. In the Zope3 codebase, these are mapped to explanatory strings
* via zodb/btrees/interfaces.py.
*/
static PyObject *
merge_error(int p1, int p2, int p3, int reason)
{
PyObject *r;
UNLESS (r=Py_BuildValue("iiii", p1, p2, p3, reason)) r=Py_None;
if (ConflictError == NULL) {
ConflictError = PyExc_ValueError;
Py_INCREF(ConflictError);
}
PyErr_SetObject(ConflictError, r);
if (r != Py_None)
{
Py_DECREF(r);
}
return NULL;
}
/* It's hard to explain "the rules" for bucket_merge, in large part because
* any automatic conflict-resolution scheme is going to be incorrect for
* some endcases of *some* app. The scheme here is pretty conservative,
* and should be OK for most apps. It's easier to explain what the code
* allows than what it forbids:
*
* Leaving things alone: it's OK if both s2 and s3 leave a piece of s1
* alone (don't delete the key, and don't change the value).
*
* Key deletion: a transaction (s2 or s3) can delete a key (from s1), but
* only if the other transaction (of s2 and s3) doesn't delete the same key.
* However, it's not OK for s2 and s3 to, between them, end up deleting all
* the keys. This is a higher-level constraint, due to that the caller of
* bucket_merge() doesn't have enough info to unlink the resulting empty
* bucket from its BTree correctly. It's also not OK if s2 or s3 are empty,
* because the transaction that emptied the bucket unlinked the bucket from
* the tree, and nothing we do here can get it linked back in again.
*
* Key insertion: s2 or s3 can add a new key, provided the other transaction
* doesn't insert the same key. It's not OK even if they insert the same
* <key, value> pair.
*
* Mapping value modification: s2 or s3 can modify the value associated
* with a key in s1, provided the other transaction doesn't make a
* modification of the same key to a different value. It's OK if s2 and s3
* both give the same new value to the key while it's hard to be precise about
* why, this doesn't seem consistent with that it's *not* OK for both to add
* a new key mapping to the same value).
*/
static PyObject *
bucket_merge(Bucket *s1, Bucket *s2, Bucket *s3)
{
Bucket *r=0;
PyObject *s;
SetIteration i1 = {0,0,0}, i2 = {0,0,0}, i3 = {0,0,0};
int cmp12, cmp13, cmp23, mapping, set;
/* If either "after" bucket is empty, punt. */
if (s2->len == 0 || s3->len == 0)
{
merge_error(-1, -1, -1, 12);
goto err;
}
if (initSetIteration(&i1, OBJECT(s1), 1) < 0)
goto err;
if (initSetIteration(&i2, OBJECT(s2), 1) < 0)
goto err;
if (initSetIteration(&i3, OBJECT(s3), 1) < 0)
goto err;
mapping = i1.usesValue | i2.usesValue | i3.usesValue;
set = !mapping;
if (mapping)
r = (Bucket *)PyObject_CallObject((PyObject *)&BucketType, NULL);
else
r = (Bucket *)PyObject_CallObject((PyObject *)&SetType, NULL);
if (r == NULL)
goto err;
if (i1.next(&i1) < 0)
goto err;
if (i2.next(&i2) < 0)
goto err;
if (i3.next(&i3) < 0)
goto err;
/* Consult zodb/btrees/interfaces.py for the meaning of the last
* argument passed to merge_error().
*/
/* TODO: This isn't passing on errors raised by value comparisons. */
while (i1.position >= 0 && i2.position >= 0 && i3.position >= 0)
{
TEST_KEY_SET_OR(cmp12, i1.key, i2.key) goto err;
TEST_KEY_SET_OR(cmp13, i1.key, i3.key) goto err;
if (cmp12==0)
{
if (cmp13==0)
{
if (set || (TEST_VALUE(i1.value, i2.value) == 0))
{ /* change in i3 value or all same */
if (merge_output(r, &i3, mapping) < 0) goto err;
}
else if (set || (TEST_VALUE(i1.value, i3.value) == 0))
{ /* change in i2 value */
if (merge_output(r, &i2, mapping) < 0) goto err;
}
else
{ /* conflicting value changes in i2 and i3 */
merge_error(i1.position, i2.position, i3.position, 1);
goto err;
}
if (i1.next(&i1) < 0) goto err;
if (i2.next(&i2) < 0) goto err;
if (i3.next(&i3) < 0) goto err;
}
else if (cmp13 > 0)
{ /* insert i3 */
if (merge_output(r, &i3, mapping) < 0) goto err;
if (i3.next(&i3) < 0) goto err;
}
else if (set || (TEST_VALUE(i1.value, i2.value) == 0))
{ /* deleted in i3 */
if (i3.position == 1)
{
/* Deleted the first item. This will modify the
parent node, so we don't know if merging will be
safe
*/
merge_error(i1.position, i2.position, i3.position, 13);
goto err;
}
if (i1.next(&i1) < 0) goto err;
if (i2.next(&i2) < 0) goto err;
}
else
{ /* conflicting del in i3 and change in i2 */
merge_error(i1.position, i2.position, i3.position, 2);
goto err;
}
}
else if (cmp13 == 0)
{
if (cmp12 > 0)
{ /* insert i2 */
if (merge_output(r, &i2, mapping) < 0) goto err;
if (i2.next(&i2) < 0) goto err;
}
else if (set || (TEST_VALUE(i1.value, i3.value) == 0))
{ /* deleted in i2 */
if (i2.position == 1)
{
/* Deleted the first item. This will modify the
parent node, so we don't know if merging will be
safe
*/
merge_error(i1.position, i2.position, i3.position, 13);
goto err;
}
if (i1.next(&i1) < 0) goto err;
if (i3.next(&i3) < 0) goto err;
}
else
{ /* conflicting del in i2 and change in i3 */
merge_error(i1.position, i2.position, i3.position, 3);
goto err;
}
}
else
{ /* Both keys changed */
TEST_KEY_SET_OR(cmp23, i2.key, i3.key) goto err;
if (cmp23==0)
{ /* dueling inserts or deletes */
merge_error(i1.position, i2.position, i3.position, 4);
goto err;
}
if (cmp12 > 0)
{ /* insert i2 */
if (cmp23 > 0)
{ /* insert i3 first */
if (merge_output(r, &i3, mapping) < 0) goto err;
if (i3.next(&i3) < 0) goto err;
}
else
{ /* insert i2 first */
if (merge_output(r, &i2, mapping) < 0) goto err;
if (i2.next(&i2) < 0) goto err;
}
}
else if (cmp13 > 0)
{ /* Insert i3 */
if (merge_output(r, &i3, mapping) < 0) goto err;
if (i3.next(&i3) < 0) goto err;
}
else
{ /* 1<2 and 1<3: both deleted 1.key */
merge_error(i1.position, i2.position, i3.position, 5);
goto err;
}
}
}
while (i2.position >= 0 && i3.position >= 0)
{ /* New inserts */
TEST_KEY_SET_OR(cmp23, i2.key, i3.key) goto err;
if (cmp23==0)
{ /* dueling inserts */
merge_error(i1.position, i2.position, i3.position, 6);
goto err;
}
if (cmp23 > 0)
{ /* insert i3 */
if (merge_output(r, &i3, mapping) < 0) goto err;
if (i3.next(&i3) < 0) goto err;
}
else
{ /* insert i2 */
if (merge_output(r, &i2, mapping) < 0) goto err;
if (i2.next(&i2) < 0) goto err;
}
}
while (i1.position >= 0 && i2.position >= 0)
{ /* remainder of i1 deleted in i3 */
TEST_KEY_SET_OR(cmp12, i1.key, i2.key) goto err;
if (cmp12 > 0)
{ /* insert i2 */
if (merge_output(r, &i2, mapping) < 0) goto err;
if (i2.next(&i2) < 0) goto err;
}
else if (cmp12==0 && (set || (TEST_VALUE(i1.value, i2.value) == 0)))
{ /* delete i3 */
if (i1.next(&i1) < 0) goto err;
if (i2.next(&i2) < 0) goto err;
}
else
{ /* Dueling deletes or delete and change */
merge_error(i1.position, i2.position, i3.position, 7);
goto err;
}
}
while (i1.position >= 0 && i3.position >= 0)
{ /* remainder of i1 deleted in i2 */
TEST_KEY_SET_OR(cmp13, i1.key, i3.key) goto err;
if (cmp13 > 0)
{ /* insert i3 */
if (merge_output(r, &i3, mapping) < 0) goto err;
if (i3.next(&i3) < 0) goto err;
}
else if (cmp13==0 && (set || (TEST_VALUE(i1.value, i3.value) == 0)))
{ /* delete i2 */
if (i1.next(&i1) < 0) goto err;
if (i3.next(&i3) < 0) goto err;
}
else
{ /* Dueling deletes or delete and change */
merge_error(i1.position, i2.position, i3.position, 8);
goto err;
}
}
if (i1.position >= 0)
{ /* Dueling deletes */
merge_error(i1.position, i2.position, i3.position, 9);
goto err;
}
while (i2.position >= 0)
{ /* Inserting i2 at end */
if (merge_output(r, &i2, mapping) < 0) goto err;
if (i2.next(&i2) < 0) goto err;
}
while (i3.position >= 0)
{ /* Inserting i3 at end */
if (merge_output(r, &i3, mapping) < 0) goto err;
if (i3.next(&i3) < 0) goto err;
}
/* If the output bucket is empty, conflict resolution doesn't have
* enough info to unlink it from its containing BTree correctly.
*/
if (r->len == 0)
{
merge_error(-1, -1, -1, 10);
goto err;
}
finiSetIteration(&i1);
finiSetIteration(&i2);
finiSetIteration(&i3);
if (s1->next)
{
Py_INCREF(s1->next);
r->next = s1->next;
}
s = bucket_getstate(r);
Py_DECREF(r);
return s;
err:
finiSetIteration(&i1);
finiSetIteration(&i2);
finiSetIteration(&i3);
Py_XDECREF(r);
return NULL;
}
BTrees/OIBTree.py 0 → 100644
View file @ 703fc7e7
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import zope.interface
import BTrees.Interfaces
# hack to overcome dynamic-linking headache.
from _OIBTree import *
zope.interface.moduleProvides(BTrees.Interfaces.IObjectIntegerBTreeModule)
BTrees/OLBTree.py 0 → 100644
View file @ 703fc7e7
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import zope.interface
import BTrees.Interfaces
# hack to overcome dynamic-linking headache.
from _OLBTree import *
zope.interface.moduleProvides(BTrees.Interfaces.IObjectIntegerBTreeModule)
BTrees/OOBTree.py 0 → 100644
View file @ 703fc7e7
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import zope.interface
import BTrees.Interfaces
# hack to overcome dynamic-linking headache.
from _OOBTree import *
zope.interface.moduleProvides(BTrees.Interfaces.IObjectObjectBTreeModule)
BTrees/SetOpTemplate.c 0 → 100644
View file @ 703fc7e7
/*****************************************************************************
Copyright (c) 2001, 2002 Zope Foundation and Contributors.
All Rights Reserved.
This software is subject to the provisions of the Zope Public License,
Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
****************************************************************************/
/****************************************************************************
Set operations
****************************************************************************/
#define SETOPTEMPLATE_C "$Id$\n"
#ifdef KEY_CHECK
static int
nextKeyAsSet(SetIteration *i)
{
if (i->position >= 0) {
if (i->position) {
DECREF_KEY(i->key);
i->position = -1;
}
else
i->position = 1;
}
return 0;
}
#endif
/* initSetIteration
*
* Start the set iteration protocol. See the comments at struct SetIteration.
*
* Arguments
* i The address of a SetIteration control struct.
* s The address of the set, bucket, BTree, ..., to be iterated.
* useValues Boolean; if true, and s has values (is a mapping), copy
* them into i->value each time i->next() is called; else
* ignore s's values even if s is a mapping.
*
* Return
* 0 on success; -1 and an exception set if error.
* i.usesValue is set to 1 (true) if s has values and useValues was
* true; else usesValue is set to 0 (false).
* i.set gets a new reference to s, or to some other object used to
* iterate over s.
* i.position is set to 0.
* i.next is set to an appropriate iteration function.
* i.key and i.value are left alone.
*
* Internal
* i.position < 0 means iteration terminated.
* i.position = 0 means iteration hasn't yet begun (next() hasn't
* been called yet).
* In all other cases, i.key, and possibly i.value, own references.
* These must be cleaned up, either by next() routines, or by
* finiSetIteration.
* next() routines must ensure the above. They should return without
* doing anything when i.position < 0.
* It's the responsibility of {init, fini}setIteration to clean up
* the reference in i.set, and to ensure that no stale references
* live in i.key or i.value if iteration terminates abnormally.
* A SetIteration struct has been cleaned up iff i.set is NULL.
*/
static int
initSetIteration(SetIteration *i, PyObject *s, int useValues)
{
i->set = NULL;
i->position = -1; /* set to 0 only on normal return */
i->usesValue = 0; /* assume it's a set or that values aren't iterated */
if (PyObject_IsInstance(s, (PyObject *)&BucketType))
{
i->set = s;
Py_INCREF(s);
if (useValues)
{
i->usesValue = 1;
i->next = nextBucket;
}
else
i->next = nextSet;
}
else if (PyObject_IsInstance(s, (PyObject *)&SetType))
{
i->set = s;
Py_INCREF(s);
i->next = nextSet;
}
else if (PyObject_IsInstance(s, (PyObject *)&BTreeType))
{
i->set = BTree_rangeSearch(BTREE(s), NULL, NULL, 'i');
UNLESS(i->set) return -1;
if (useValues)
{
i->usesValue = 1;
i->next = nextBTreeItems;
}
else
i->next = nextTreeSetItems;
}
else if (PyObject_IsInstance(s, (PyObject *)&TreeSetType))
{
i->set = BTree_rangeSearch(BTREE(s), NULL, NULL, 'k');
UNLESS(i->set) return -1;
i->next = nextTreeSetItems;
}
#ifdef KEY_CHECK
else if (KEY_CHECK(s))
{
int copied = 1;
COPY_KEY_FROM_ARG(i->key, s, copied);
UNLESS (copied) return -1;
INCREF_KEY(i->key);
i->set = s;
Py_INCREF(s);
i->next = nextKeyAsSet;
}
#endif
else
{
PyErr_SetString(PyExc_TypeError, "invalid argument");
return -1;
}
i->position = 0;
return 0;
}
#ifndef MERGE_WEIGHT
#define MERGE_WEIGHT(O, w) (O)
#endif
static int
copyRemaining(Bucket *r, SetIteration *i, int merge,
/* See comment # 42 */
#ifdef MERGE
VALUE_TYPE w)
#else
int w)
#endif
{
while (i->position >= 0)
{
if(r->len >= r->size && Bucket_grow(r, -1, ! merge) < 0) return -1;
COPY_KEY(r->keys[r->len], i->key);
INCREF_KEY(r->keys[r->len]);
if (merge)
{
COPY_VALUE(r->values[r->len], MERGE_WEIGHT(i->value, w));
INCREF_VALUE(r->values[r->len]);
}
r->len++;
if (i->next(i) < 0) return -1;
}
return 0;
}
/* This is the workhorse for all set merge operations: the weighted and
* unweighted flavors of union and intersection, and set difference. The
* algorithm is conceptually simple but the code is complicated due to all
* the options.
*
* s1, s2
* The input collections to be merged.
*
* usevalues1, usevalues2
* Booleans. In the output, should values from s1 (or s2) be used? This
* only makes sense when an operation intends to support mapping outputs;
* these should both be false for operations that want pure set outputs.
*
* w1, w2
* If usevalues1(2) are true, these are the weights to apply to the
* input values.
*
* c1
* Boolean. Should keys that appear in c1 but not c2 appear in the output?
* c12
* Boolean. Should keys that appear in both inputs appear in the output?
* c2
* Boolean. Should keys that appear in c2 but not c1 appear in the output?
*
* Returns NULL if error, else a Set or Bucket, depending on whether a set or
* mapping was requested.
*/
static PyObject *
set_operation(PyObject *s1, PyObject *s2,
int usevalues1, int usevalues2,
/* Comment # 42
The following ifdef works around a template/type problem
Weights are passed as integers. In particular, the weight passed by
difference is one. This works fine in the int value and float value
cases but makes no sense in the object value case. In the object
value case, we don't do merging, so we don't use the weights, so it
doesn't matter what they are.
*/
#ifdef MERGE
VALUE_TYPE w1, VALUE_TYPE w2,
#else
int w1, int w2,
#endif
int c1, int c12, int c2)
{
Bucket *r=0;
SetIteration i1 = {0,0,0}, i2 = {0,0,0};
int cmp, merge;
if (initSetIteration(&i1, s1, usevalues1) < 0) goto err;
if (initSetIteration(&i2, s2, usevalues2) < 0) goto err;
merge = i1.usesValue | i2.usesValue;
if (merge)
{
#ifndef MERGE
if (c12 && i1.usesValue && i2.usesValue) goto invalid_set_operation;
#endif
if (! i1.usesValue&& i2.usesValue)
{
SetIteration t;
int i;
/* See comment # 42 above */
#ifdef MERGE
VALUE_TYPE v;
#else
int v;
#endif
t=i1; i1=i2; i2=t;
i=c1; c1=c2; c2=i;
v=w1; w1=w2; w2=v;
}
#ifdef MERGE_DEFAULT
i1.value=MERGE_DEFAULT;
i2.value=MERGE_DEFAULT;
#else
if (i1.usesValue)
{
if (! i2.usesValue && c2) goto invalid_set_operation;
}
else
{
if (c1 || c12) goto invalid_set_operation;
}
#endif
UNLESS(r=BUCKET(PyObject_CallObject(OBJECT(&BucketType), NULL)))
goto err;
}
else
{
UNLESS(r=BUCKET(PyObject_CallObject(OBJECT(&SetType), NULL)))
goto err;
}
if (i1.next(&i1) < 0) goto err;
if (i2.next(&i2) < 0) goto err;
while (i1.position >= 0 && i2.position >= 0)
{
TEST_KEY_SET_OR(cmp, i1.key, i2.key) goto err;
if(cmp < 0)
{
if(c1)
{
if(r->len >= r->size && Bucket_grow(r, -1, ! merge) < 0) goto err;
COPY_KEY(r->keys[r->len], i1.key);
INCREF_KEY(r->keys[r->len]);
if (merge)
{
COPY_VALUE(r->values[r->len], MERGE_WEIGHT(i1.value, w1));
INCREF_VALUE(r->values[r->len]);
}
r->len++;
}
if (i1.next(&i1) < 0) goto err;
}
else if(cmp==0)
{
if(c12)
{
if(r->len >= r->size && Bucket_grow(r, -1, ! merge) < 0) goto err;
COPY_KEY(r->keys[r->len], i1.key);
INCREF_KEY(r->keys[r->len]);
if (merge)
{
#ifdef MERGE
r->values[r->len] = MERGE(i1.value, w1, i2.value, w2);
#else
COPY_VALUE(r->values[r->len], i1.value);
INCREF_VALUE(r->values[r->len]);
#endif
}
r->len++;
}
if (i1.next(&i1) < 0) goto err;
if (i2.next(&i2) < 0) goto err;
}
else
{
if(c2)
{
if(r->len >= r->size && Bucket_grow(r, -1, ! merge) < 0) goto err;
COPY_KEY(r->keys[r->len], i2.key);
INCREF_KEY(r->keys[r->len]);
if (merge)
{
COPY_VALUE(r->values[r->len], MERGE_WEIGHT(i2.value, w2));
INCREF_VALUE(r->values[r->len]);
}
r->len++;
}
if (i2.next(&i2) < 0) goto err;
}
}
if(c1 && copyRemaining(r, &i1, merge, w1) < 0) goto err;
if(c2 && copyRemaining(r, &i2, merge, w2) < 0) goto err;
finiSetIteration(&i1);
finiSetIteration(&i2);
return OBJECT(r);
#ifndef MERGE_DEFAULT
invalid_set_operation:
PyErr_SetString(PyExc_TypeError, "invalid set operation");
#endif
err:
finiSetIteration(&i1);
finiSetIteration(&i2);
Py_XDECREF(r);
return NULL;
}
static PyObject *
difference_m(PyObject *ignored, PyObject *args)
{
PyObject *o1, *o2;
UNLESS(PyArg_ParseTuple(args, "OO", &o1, &o2)) return NULL;
if (o1 == Py_None || o2 == Py_None)
{
/* difference(None, X) -> None; difference(X, None) -> X */
Py_INCREF(o1);
return o1;
}
return set_operation(o1, o2, 1, 0, /* preserve values from o1, ignore o2's */
1, 0, /* o1's values multiplied by 1 */
1, 0, 0); /* take only keys unique to o1 */
}
static PyObject *
union_m(PyObject *ignored, PyObject *args)
{
PyObject *o1, *o2;
UNLESS(PyArg_ParseTuple(args, "OO", &o1, &o2)) return NULL;
if (o1 == Py_None)
{
Py_INCREF(o2);
return o2;
}
else if (o2 == Py_None)
{
Py_INCREF(o1);
return o1;
}
return set_operation(o1, o2, 0, 0, /* ignore values in both */
1, 1, /* the weights are irrelevant */
1, 1, 1); /* take all keys */
}
static PyObject *
intersection_m(PyObject *ignored, PyObject *args)
{
PyObject *o1, *o2;
UNLESS(PyArg_ParseTuple(args, "OO", &o1, &o2)) return NULL;
if (o1 == Py_None)
{
Py_INCREF(o2);
return o2;
}
else if (o2 == Py_None)
{
Py_INCREF(o1);
return o1;
}
return set_operation(o1, o2, 0, 0, /* ignore values in both */
1, 1, /* the weights are irrelevant */
0, 1, 0); /* take only keys common to both */
}
#ifdef MERGE
static PyObject *
wunion_m(PyObject *ignored, PyObject *args)
{
PyObject *o1, *o2;
VALUE_TYPE w1 = 1, w2 = 1;
UNLESS(PyArg_ParseTuple(args, "OO|" VALUE_PARSE VALUE_PARSE,
&o1, &o2, &w1, &w2)
) return NULL;
if (o1 == Py_None)
return Py_BuildValue(VALUE_PARSE "O", (o2 == Py_None ? 0 : w2), o2);
else if (o2 == Py_None)
return Py_BuildValue(VALUE_PARSE "O", w1, o1);
o1 = set_operation(o1, o2, 1, 1, w1, w2, 1, 1, 1);
if (o1)
ASSIGN(o1, Py_BuildValue(VALUE_PARSE "O", (VALUE_TYPE)1, o1));
return o1;
}
static PyObject *
wintersection_m(PyObject *ignored, PyObject *args)
{
PyObject *o1, *o2;
VALUE_TYPE w1 = 1, w2 = 1;
UNLESS(PyArg_ParseTuple(args, "OO|" VALUE_PARSE VALUE_PARSE,
&o1, &o2, &w1, &w2)
) return NULL;
if (o1 == Py_None)
return Py_BuildValue(VALUE_PARSE "O", (o2 == Py_None ? 0 : w2), o2);
else if (o2 == Py_None)
return Py_BuildValue(VALUE_PARSE "O", w1, o1);
o1 = set_operation(o1, o2, 1, 1, w1, w2, 0, 1, 0);
if (o1)
ASSIGN(o1, Py_BuildValue(VALUE_PARSE "O",
((o1->ob_type == (PyTypeObject*)(&SetType)) ? w2+w1 : 1),
o1));
return o1;
}
#endif
#ifdef MULTI_INT_UNION
#include "sorters.c"
/* Input is a sequence of integer sets (or convertible to sets by the
set iteration protocol). Output is the union of the sets. The point
is to run much faster than doing pairs of unions.
*/
static PyObject *
multiunion_m(PyObject *ignored, PyObject *args)
{
PyObject *seq; /* input sequence */
int n; /* length of input sequence */
PyObject *set = NULL; /* an element of the input sequence */
Bucket *result; /* result set */
SetIteration setiter = {0};
int i;
UNLESS(PyArg_ParseTuple(args, "O", &seq))
return NULL;
n = PyObject_Length(seq);
if (n < 0)
return NULL;
/* Construct an empty result set. */
result = BUCKET(PyObject_CallObject(OBJECT(&SetType), NULL));
if (result == NULL)
return NULL;
/* For each set in the input sequence, append its elements to the result
set. At this point, we ignore the possibility of duplicates. */
for (i = 0; i < n; ++i) {
set = PySequence_GetItem(seq, i);
if (set == NULL)
goto Error;
/* If set is a bucket, do a straight resize + memcpy. */
if (set->ob_type == (PyTypeObject*)&SetType ||
set->ob_type == (PyTypeObject*)&BucketType)
{
Bucket *b = BUCKET(set);
int status = 0;
UNLESS (PER_USE(b)) goto Error;
if (b->len)
status = bucket_append(result, b, 0, b->len, 0, i < n-1);
PER_UNUSE(b);
if (status < 0) goto Error;
}
else {
/* No cheap way: iterate over set's elements one at a time. */
if (initSetIteration(&setiter, set, 0) < 0) goto Error;
if (setiter.next(&setiter) < 0) goto Error;
while (setiter.position >= 0) {
if (result->len >= result->size && Bucket_grow(result, -1, 1) < 0)
goto Error;
COPY_KEY(result->keys[result->len], setiter.key);
++result->len;
/* We know the key is an int, so no need to incref it. */
if (setiter.next(&setiter) < 0) goto Error;
}
finiSetIteration(&setiter);
}
Py_DECREF(set);
set = NULL;
}
/* Combine, sort, remove duplicates, and reset the result's len.
If the set shrinks (which happens if and only if there are
duplicates), no point to realloc'ing the set smaller, as we
expect the result set to be short-lived.
*/
if (result->len > 0) {
size_t newlen; /* number of elements in final result set */
newlen = sort_int_nodups(result->keys, (size_t)result->len);
result->len = (int)newlen;
}
return (PyObject *)result;
Error:
Py_DECREF(result);
Py_XDECREF(set);
finiSetIteration(&setiter);
return NULL;
}
#endif
BTrees/SetTemplate.c 0 → 100644
View file @ 703fc7e7
/*****************************************************************************
Copyright (c) 2001, 2002 Zope Foundation and Contributors.
All Rights Reserved.
This software is subject to the provisions of the Zope Public License,
Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
****************************************************************************/
#define SETTEMPLATE_C "$Id$\n"
static PyObject *
Set_insert(Bucket *self, PyObject *args)
{
PyObject *key;
int i;
UNLESS (PyArg_ParseTuple(args, "O", &key)) return NULL;
if ( (i=_bucket_set(self, key, Py_None, 1, 1, 0)) < 0) return NULL;
return PyInt_FromLong(i);
}
/* _Set_update and _TreeSet_update are identical except for the
function they call to add the element to the set.
*/
static int
_Set_update(Bucket *self, PyObject *seq)
{
int n=0, ind=0;
PyObject *iter, *v;
iter = PyObject_GetIter(seq);
if (iter == NULL)
return -1;
while (1) {
v = PyIter_Next(iter);
if (v == NULL) {
if (PyErr_Occurred())
goto err;
else
break;
}
ind = _bucket_set(self, v, Py_None, 1, 1, 0);
Py_DECREF(v);
if (ind < 0)
goto err;
else
n += ind;
}
err:
Py_DECREF(iter);
if (ind < 0)
return -1;
return n;
}
static PyObject *
Set_update(Bucket *self, PyObject *args)
{
PyObject *seq = NULL;
int n = 0;
if (!PyArg_ParseTuple(args, "|O:update", &seq))
return NULL;
if (seq) {
n = _Set_update(self, seq);
if (n < 0)
return NULL;
}
return PyInt_FromLong(n);
}
static PyObject *
Set_remove(Bucket *self, PyObject *args)
{
PyObject *key;
UNLESS (PyArg_ParseTuple(args, "O", &key)) return NULL;
if (_bucket_set(self, key, NULL, 0, 1, 0) < 0) return NULL;
Py_INCREF(Py_None);
return Py_None;
}
static int
_set_setstate(Bucket *self, PyObject *args)
{
PyObject *k, *items;
Bucket *next=0;
int i, l, copied=1;
KEY_TYPE *keys;
UNLESS (PyArg_ParseTuple(args, "O|O", &items, &next))
return -1;
if (!PyTuple_Check(items)) {
PyErr_SetString(PyExc_TypeError,
"tuple required for first state element");
return -1;
}
if ((l=PyTuple_Size(items)) < 0) return -1;
for (i=self->len; --i >= 0; )
{
DECREF_KEY(self->keys[i]);
}
self->len=0;
if (self->next)
{
Py_DECREF(self->next);
self->next=0;
}
if (l > self->size)
{
UNLESS (keys=BTree_Realloc(self->keys, sizeof(KEY_TYPE)*l)) return -1;
self->keys=keys;
self->size=l;
}
for (i=0; i<l; i++)
{
k=PyTuple_GET_ITEM(items, i);
COPY_KEY_FROM_ARG(self->keys[i], k, copied);
UNLESS (copied) return -1;
INCREF_KEY(self->keys[i]);
}
self->len=l;
if (next)
{
self->next=next;
Py_INCREF(next);
}
return 0;
}
static PyObject *
set_setstate(Bucket *self, PyObject *args)
{
int r;
UNLESS (PyArg_ParseTuple(args, "O", &args)) return NULL;
PER_PREVENT_DEACTIVATION(self);
r=_set_setstate(self, args);
PER_UNUSE(self);
if (r < 0) return NULL;
Py_INCREF(Py_None);
return Py_None;
}
static struct PyMethodDef Set_methods[] = {
{"__getstate__", (PyCFunction) bucket_getstate, METH_VARARGS,
"__getstate__() -- Return the picklable state of the object"},
{"__setstate__", (PyCFunction) set_setstate, METH_VARARGS,
"__setstate__() -- Set the state of the object"},
{"keys", (PyCFunction) bucket_keys, METH_KEYWORDS,
"keys() -- Return the keys"},
{"has_key", (PyCFunction) bucket_has_key, METH_O,
"has_key(key) -- Test whether the bucket contains the given key"},
{"clear", (PyCFunction) bucket_clear, METH_VARARGS,
"clear() -- Remove all of the items from the bucket"},
{"maxKey", (PyCFunction) Bucket_maxKey, METH_VARARGS,
"maxKey([key]) -- Find the maximum key\n\n"
"If an argument is given, find the maximum <= the argument"},
{"minKey", (PyCFunction) Bucket_minKey, METH_VARARGS,
"minKey([key]) -- Find the minimum key\n\n"
"If an argument is given, find the minimum >= the argument"},
#ifdef PERSISTENT
{"_p_resolveConflict", (PyCFunction) bucket__p_resolveConflict, METH_VARARGS,
"_p_resolveConflict() -- Reinitialize from a newly created copy"},
{"_p_deactivate", (PyCFunction) bucket__p_deactivate, METH_KEYWORDS,
"_p_deactivate() -- Reinitialize from a newly created copy"},
#endif
{"add", (PyCFunction)Set_insert, METH_VARARGS,
"add(id) -- Add a key to the set"},
{"insert", (PyCFunction)Set_insert, METH_VARARGS,
"insert(id) -- Add a key to the set"},
{"update", (PyCFunction)Set_update, METH_VARARGS,
"update(seq) -- Add the items from the given sequence to the set"},
{"remove", (PyCFunction)Set_remove, METH_VARARGS,
"remove(id) -- Remove an id from the set"},
{NULL, NULL} /* sentinel */
};
static int
Set_init(PyObject *self, PyObject *args, PyObject *kwds)
{
PyObject *v = NULL;
if (!PyArg_ParseTuple(args, "|O:" MOD_NAME_PREFIX "Set", &v))
return -1;
if (v)
return _Set_update((Bucket *)self, v);
else
return 0;
}
static PyObject *
set_repr(Bucket *self)
{
static PyObject *format;
PyObject *r, *t;
if (!format)
format = PyString_FromString(MOD_NAME_PREFIX "Set(%s)");
UNLESS (t = PyTuple_New(1)) return NULL;
UNLESS (r = bucket_keys(self, NULL, NULL)) goto err;
PyTuple_SET_ITEM(t, 0, r);
r = t;
ASSIGN(r, PyString_Format(format, r));
return r;
err:
Py_DECREF(t);
return NULL;
}
static Py_ssize_t
set_length(Bucket *self)
{
int r;
PER_USE_OR_RETURN(self, -1);
r = self->len;
PER_UNUSE(self);
return r;
}
static PyObject *
set_item(Bucket *self, Py_ssize_t index)
{
PyObject *r=0;
PER_USE_OR_RETURN(self, NULL);
if (index >= 0 && index < self->len)
{
COPY_KEY_TO_OBJECT(r, self->keys[index]);
}
else
IndexError(index);
PER_UNUSE(self);
return r;
}
static PySequenceMethods set_as_sequence = {
(lenfunc)set_length, /* sq_length */
(binaryfunc)0, /* sq_concat */
(ssizeargfunc)0, /* sq_repeat */
(ssizeargfunc)set_item, /* sq_item */
(ssizessizeargfunc)0, /* sq_slice */
(ssizeobjargproc)0, /* sq_ass_item */
(ssizessizeobjargproc)0, /* sq_ass_slice */
(objobjproc)bucket_contains, /* sq_contains */
0, /* sq_inplace_concat */
0, /* sq_inplace_repeat */
};
static PyTypeObject SetType = {
PyObject_HEAD_INIT(NULL) /* PyPersist_Type */
0, /* ob_size */
MODULE_NAME MOD_NAME_PREFIX "Set", /* tp_name */
sizeof(Bucket), /* tp_basicsize */
0, /* tp_itemsize */
(destructor)bucket_dealloc, /* tp_dealloc */
0, /* tp_print */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_compare */
(reprfunc)set_repr, /* tp_repr */
0, /* tp_as_number */
&set_as_sequence, /* tp_as_sequence */
0, /* tp_as_mapping */
0, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
0, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
Py_TPFLAGS_BASETYPE, /* tp_flags */
0, /* tp_doc */
(traverseproc)bucket_traverse, /* tp_traverse */
(inquiry)bucket_tp_clear, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
(getiterfunc)Bucket_getiter, /* tp_iter */
0, /* tp_iternext */
Set_methods, /* tp_methods */
Bucket_members, /* tp_members */
0, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
Set_init, /* tp_init */
0, /* tp_alloc */
0, /*PyType_GenericNew,*/ /* tp_new */
};
static int
nextSet(SetIteration *i)
{
if (i->position >= 0)
{
UNLESS(PER_USE(BUCKET(i->set))) return -1;
if (i->position)
{
DECREF_KEY(i->key);
}
if (i->position < BUCKET(i->set)->len)
{
COPY_KEY(i->key, BUCKET(i->set)->keys[i->position]);
INCREF_KEY(i->key);
i->position ++;
}
else
{
i->position = -1;
PER_ACCESSED(BUCKET(i->set));
}
PER_ALLOW_DEACTIVATION(BUCKET(i->set));
}
return 0;
}
BTrees/TreeSetTemplate.c 0 → 100644
View file @ 703fc7e7
/*****************************************************************************
Copyright (c) 2001, 2002 Zope Foundation and Contributors.
All Rights Reserved.
This software is subject to the provisions of the Zope Public License,
Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
****************************************************************************/
#define TREESETTEMPLATE_C "$Id$\n"
static PyObject *
TreeSet_insert(BTree *self, PyObject *args)
{
PyObject *key;
int i;
if (!PyArg_ParseTuple(args, "O:insert", &key))
return NULL;
i = _BTree_set(self, key, Py_None, 1, 1);
if (i < 0)
return NULL;
return PyInt_FromLong(i);
}
/* _Set_update and _TreeSet_update are identical except for the
function they call to add the element to the set.
*/
static int
_TreeSet_update(BTree *self, PyObject *seq)
{
int n=0, ind=0;
PyObject *iter, *v;
iter = PyObject_GetIter(seq);
if (iter == NULL)
return -1;
while (1) {
v = PyIter_Next(iter);
if (v == NULL) {
if (PyErr_Occurred())
goto err;
else
break;
}
ind = _BTree_set(self, v, Py_None, 1, 1);
Py_DECREF(v);
if (ind < 0)
goto err;
else
n += ind;
}
err:
Py_DECREF(iter);
if (ind < 0)
return -1;
return n;
}
static PyObject *
TreeSet_update(BTree *self, PyObject *args)
{
PyObject *seq = NULL;
int n = 0;
if (!PyArg_ParseTuple(args, "|O:update", &seq))
return NULL;
if (seq) {
n = _TreeSet_update(self, seq);
if (n < 0)
return NULL;
}
return PyInt_FromLong(n);
}
static PyObject *
TreeSet_remove(BTree *self, PyObject *args)
{
PyObject *key;
UNLESS (PyArg_ParseTuple(args, "O", &key)) return NULL;
if (_BTree_set(self, key, NULL, 0, 1) < 0) return NULL;
Py_INCREF(Py_None);
return Py_None;
}
static PyObject *
TreeSet_setstate(BTree *self, PyObject *args)
{
int r;
if (!PyArg_ParseTuple(args,"O",&args)) return NULL;
PER_PREVENT_DEACTIVATION(self);
r=_BTree_setstate(self, args, 1);
PER_UNUSE(self);
if (r < 0) return NULL;
Py_INCREF(Py_None);
return Py_None;
}
static struct PyMethodDef TreeSet_methods[] = {
{"__getstate__", (PyCFunction) BTree_getstate, METH_NOARGS,
"__getstate__() -> state\n\n"
"Return the picklable state of the TreeSet."},
{"__setstate__", (PyCFunction) TreeSet_setstate, METH_VARARGS,
"__setstate__(state)\n\n"
"Set the state of the TreeSet."},
{"has_key", (PyCFunction) BTree_has_key, METH_O,
"has_key(key)\n\n"
"Return true if the TreeSet contains the given key."},
{"keys", (PyCFunction) BTree_keys, METH_KEYWORDS,
"keys([min, max]) -> list of keys\n\n"
"Returns the keys of the TreeSet. If min and max are supplied, only\n"
"keys greater than min and less than max are returned."},
{"maxKey", (PyCFunction) BTree_maxKey, METH_VARARGS,
"maxKey([max]) -> key\n\n"
"Return the largest key in the BTree. If max is specified, return\n"
"the largest key <= max."},
{"minKey", (PyCFunction) BTree_minKey, METH_VARARGS,
"minKey([mi]) -> key\n\n"
"Return the smallest key in the BTree. If min is specified, return\n"
"the smallest key >= min."},
{"clear", (PyCFunction) BTree_clear, METH_NOARGS,
"clear()\n\nRemove all of the items from the BTree."},
{"add", (PyCFunction)TreeSet_insert, METH_VARARGS,
"add(id) -- Add an item to the set"},
{"insert", (PyCFunction)TreeSet_insert, METH_VARARGS,
"insert(id) -- Add an item to the set"},
{"update", (PyCFunction)TreeSet_update, METH_VARARGS,
"update(collection)\n\n Add the items from the given collection."},
{"remove", (PyCFunction)TreeSet_remove, METH_VARARGS,
"remove(id) -- Remove a key from the set"},
{"_check", (PyCFunction) BTree_check, METH_NOARGS,
"Perform sanity check on TreeSet, and raise exception if flawed."},
#ifdef PERSISTENT
{"_p_resolveConflict", (PyCFunction) BTree__p_resolveConflict, METH_VARARGS,
"_p_resolveConflict() -- Reinitialize from a newly created copy"},
{"_p_deactivate", (PyCFunction) BTree__p_deactivate, METH_KEYWORDS,
"_p_deactivate()\n\nReinitialize from a newly created copy."},
#endif
{NULL, NULL} /* sentinel */
};
static PyMappingMethods TreeSet_as_mapping = {
(lenfunc)BTree_length, /*mp_length*/
};
static PySequenceMethods TreeSet_as_sequence = {
(lenfunc)0, /* sq_length */
(binaryfunc)0, /* sq_concat */
(ssizeargfunc)0, /* sq_repeat */
(ssizeargfunc)0, /* sq_item */
(ssizessizeargfunc)0, /* sq_slice */
(ssizeobjargproc)0, /* sq_ass_item */
(ssizessizeobjargproc)0, /* sq_ass_slice */
(objobjproc)BTree_contains, /* sq_contains */
0, /* sq_inplace_concat */
0, /* sq_inplace_repeat */
};
static int
TreeSet_init(PyObject *self, PyObject *args, PyObject *kwds)
{
PyObject *v = NULL;
if (!PyArg_ParseTuple(args, "|O:" MOD_NAME_PREFIX "TreeSet", &v))
return -1;
if (v)
return _TreeSet_update((BTree *)self, v);
else
return 0;
}
static PyTypeObject TreeSetType = {
PyObject_HEAD_INIT(NULL) /* PyPersist_Type */
0, /* ob_size */
MODULE_NAME MOD_NAME_PREFIX "TreeSet",/* tp_name */
sizeof(BTree), /* tp_basicsize */
0, /* tp_itemsize */
(destructor)BTree_dealloc, /* tp_dealloc */
0, /* tp_print */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_compare */
0, /* tp_repr */
&BTree_as_number_for_nonzero, /* tp_as_number */
&TreeSet_as_sequence, /* tp_as_sequence */
&TreeSet_as_mapping, /* tp_as_mapping */
0, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
0, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
Py_TPFLAGS_BASETYPE, /* tp_flags */
0, /* tp_doc */
(traverseproc)BTree_traverse, /* tp_traverse */
(inquiry)BTree_tp_clear, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
(getiterfunc)BTree_getiter, /* tp_iter */
0, /* tp_iternext */
TreeSet_methods, /* tp_methods */
BTree_members, /* tp_members */
0, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
TreeSet_init, /* tp_init */
0, /* tp_alloc */
0, /*PyType_GenericNew,*/ /* tp_new */
};
BTrees/_IFBTree.c 0 → 100644
View file @ 703fc7e7
/*############################################################################
#
# Copyright (c) 2004 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
############################################################################*/
#define MASTER_ID "$Id$\n"
/* IFBTree - int key, float value BTree
Implements a collection using int type keys
and float type values
*/
/* Setup template macros */
#define PERSISTENT
#define MOD_NAME_PREFIX "IF"
#define INITMODULE init_IFBTree
#define DEFAULT_MAX_BUCKET_SIZE 120
#define DEFAULT_MAX_BTREE_SIZE 500
#include "intkeymacros.h"
#include "floatvaluemacros.h"
#include "BTreeModuleTemplate.c"
BTrees/_IIBTree.c 0 → 100644
View file @ 703fc7e7
/*############################################################################
#
# Copyright (c) 2004 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
############################################################################*/
#define MASTER_ID "$Id$\n"
/* IIBTree - int key, int value BTree
Implements a collection using int type keys
and int type values
*/
/* Setup template macros */
#define PERSISTENT
#define MOD_NAME_PREFIX "II"
#define INITMODULE init_IIBTree
#define DEFAULT_MAX_BUCKET_SIZE 120
#define DEFAULT_MAX_BTREE_SIZE 500
#include "intkeymacros.h"
#include "intvaluemacros.h"
#include "BTreeModuleTemplate.c"
BTrees/_IOBTree.c 0 → 100644
View file @ 703fc7e7
/*############################################################################
#
# Copyright (c) 2004 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
############################################################################*/
#define MASTER_ID "$Id$\n"
/* IOBTree - int key, object value BTree
Implements a collection using int type keys
and object type values
*/
#define PERSISTENT
#define MOD_NAME_PREFIX "IO"
#define DEFAULT_MAX_BUCKET_SIZE 60
#define DEFAULT_MAX_BTREE_SIZE 500
#define INITMODULE init_IOBTree
#include "intkeymacros.h"
#include "objectvaluemacros.h"
#include "BTreeModuleTemplate.c"
BTrees/_LFBTree.c 0 → 100644
View file @ 703fc7e7
/*############################################################################
#
# Copyright (c) 2004 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
############################################################################*/
#define MASTER_ID "$Id: _IFBTree.c 67074 2006-04-17 19:13:39Z fdrake $\n"
/* IFBTree - int key, float value BTree
Implements a collection using int type keys
and float type values
*/
/* Setup template macros */
#define PERSISTENT
#define MOD_NAME_PREFIX "LF"
#define INITMODULE init_LFBTree
#define DEFAULT_MAX_BUCKET_SIZE 120
#define DEFAULT_MAX_BTREE_SIZE 500
#define ZODB_64BIT_INTS
#include "intkeymacros.h"
#include "floatvaluemacros.h"
#include "BTreeModuleTemplate.c"
BTrees/_LLBTree.c 0 → 100644
View file @ 703fc7e7
/*############################################################################
#
# Copyright (c) 2004 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
############################################################################*/
#define MASTER_ID "$Id: _IIBTree.c 25186 2004-06-02 15:07:33Z jim $\n"
/* IIBTree - int key, int value BTree
Implements a collection using int type keys
and int type values
*/
/* Setup template macros */
#define PERSISTENT
#define MOD_NAME_PREFIX "LL"
#define INITMODULE init_LLBTree
#define DEFAULT_MAX_BUCKET_SIZE 120
#define DEFAULT_MAX_BTREE_SIZE 500
#define ZODB_64BIT_INTS
#include "intkeymacros.h"
#include "intvaluemacros.h"
#include "BTreeModuleTemplate.c"
BTrees/_LOBTree.c 0 → 100644
View file @ 703fc7e7
/*############################################################################
#
# Copyright (c) 2004 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
############################################################################*/
#define MASTER_ID "$Id: _IOBTree.c 25186 2004-06-02 15:07:33Z jim $\n"
/* IOBTree - int key, object value BTree
Implements a collection using int type keys
and object type values
*/
#define PERSISTENT
#define MOD_NAME_PREFIX "LO"
#define DEFAULT_MAX_BUCKET_SIZE 60
#define DEFAULT_MAX_BTREE_SIZE 500
#define INITMODULE init_LOBTree
#define ZODB_64BIT_INTS
#include "intkeymacros.h"
#include "objectvaluemacros.h"
#include "BTreeModuleTemplate.c"
BTrees/_OIBTree.c 0 → 100644
View file @ 703fc7e7
/*############################################################################
#
# Copyright (c) 2004 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
############################################################################*/
#define MASTER_ID "$Id$\n"
/* OIBTree - object key, int value BTree
Implements a collection using object type keys
and int type values
*/
#define PERSISTENT
#define MOD_NAME_PREFIX "OI"
#define INITMODULE init_OIBTree
#define DEFAULT_MAX_BUCKET_SIZE 60
#define DEFAULT_MAX_BTREE_SIZE 250
#include "objectkeymacros.h"
#include "intvaluemacros.h"
#include "BTreeModuleTemplate.c"
BTrees/_OLBTree.c 0 → 100644
View file @ 703fc7e7
/*############################################################################
#
# Copyright (c) 2004 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
############################################################################*/
#define MASTER_ID "$Id: _OIBTree.c 25186 2004-06-02 15:07:33Z jim $\n"
/* OIBTree - object key, int value BTree
Implements a collection using object type keys
and int type values
*/
#define PERSISTENT
#define MOD_NAME_PREFIX "OL"
#define INITMODULE init_OLBTree
#define DEFAULT_MAX_BUCKET_SIZE 60
#define DEFAULT_MAX_BTREE_SIZE 250
#define ZODB_64BIT_INTS
#include "objectkeymacros.h"
#include "intvaluemacros.h"
#include "BTreeModuleTemplate.c"
BTrees/_OOBTree.c 0 → 100644
View file @ 703fc7e7
/*############################################################################
#
# Copyright (c) 2004 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
############################################################################*/
#define MASTER_ID "$Id$\n"
/* OOBTree - object key, object value BTree
Implements a collection using object type keys
and object type values
*/
#define PERSISTENT
#define MOD_NAME_PREFIX "OO"
#define INITMODULE init_OOBTree
#define DEFAULT_MAX_BUCKET_SIZE 30
#define DEFAULT_MAX_BTREE_SIZE 250
#include "objectkeymacros.h"
#include "objectvaluemacros.h"
#include "BTreeModuleTemplate.c"
BTrees/__init__.py 0 → 100644
View file @ 703fc7e7
#############################################################################
#
# Copyright (c) 2007 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
#############################################################################
import zope.interface
import BTrees.Interfaces
@zope.interface.implementer(BTrees.Interfaces.IBTreeFamily)
class _Family(object):
from BTrees import OOBTree as OO
class _Family32(_Family):
from BTrees import OIBTree as OI
from BTrees import IIBTree as II
from BTrees import IOBTree as IO
from BTrees import IFBTree as IF
maxint = int(2**31-1)
minint = -maxint - 1
def __reduce__(self):
return _family32, ()
class _Family64(_Family):
from BTrees import OLBTree as OI
from BTrees import LLBTree as II
from BTrees import LOBTree as IO
from BTrees import LFBTree as IF
maxint = 2**63-1
minint = -maxint - 1
def __reduce__(self):
return _family64, ()
def _family32():
return family32
_family32.__safe_for_unpickling__ = True
def _family64():
return family64
_family64.__safe_for_unpickling__ = True
family32 = _Family32()
family64 = _Family64()
BTrees.family64.IO.family = family64
BTrees.family64.OI.family = family64
BTrees.family64.IF.family = family64
BTrees.family64.II.family = family64
BTrees.family32.IO.family = family32
BTrees.family32.OI.family = family32
BTrees.family32.IF.family = family32
BTrees.family32.II.family = family32
BTrees/_fsBTree.c 0 → 100644
View file @ 703fc7e7
/*############################################################################
#
# Copyright (c) 2004 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
############################################################################*/
#define MASTER_ID "$Id$\n"
/* fsBTree - FileStorage index BTree
This BTree implements a mapping from 2-character strings
to six-character strings. This allows us to efficiently store
a FileStorage index as a nested mapping of 6-character oid prefix
to mapping of 2-character oid suffix to 6-character (byte) file
positions.
*/
typedef unsigned char char2[2];
typedef unsigned char char6[6];
/* Setup template macros */
#define PERSISTENT
#define MOD_NAME_PREFIX "fs"
#define INITMODULE init_fsBTree
#define DEFAULT_MAX_BUCKET_SIZE 500
#define DEFAULT_MAX_BTREE_SIZE 500
/*#include "intkeymacros.h"*/
#define KEYMACROS_H "$Id$\n"
#define KEY_TYPE char2
#undef KEY_TYPE_IS_PYOBJECT
#define KEY_CHECK(K) (PyString_Check(K) && PyString_GET_SIZE(K)==2)
#define TEST_KEY_SET_OR(V, K, T) if ( ( (V) = ((*(K) < *(T) || (*(K) == *(T) && (K)[1] < (T)[1])) ? -1 : ((*(K) == *(T) && (K)[1] == (T)[1]) ? 0 : 1)) ), 0 )
#define DECREF_KEY(KEY)
#define INCREF_KEY(k)
#define COPY_KEY(KEY, E) (*(KEY)=*(E), (KEY)[1]=(E)[1])
#define COPY_KEY_TO_OBJECT(O, K) O=PyString_FromStringAndSize((const char*)K,2)
#define COPY_KEY_FROM_ARG(TARGET, ARG, STATUS) \
if (KEY_CHECK(ARG)) memcpy(TARGET, PyString_AS_STRING(ARG), 2); else { \
PyErr_SetString(PyExc_TypeError, "expected two-character string key"); \
(STATUS)=0; }
/*#include "intvaluemacros.h"*/
#define VALUEMACROS_H "$Id$\n"
#define VALUE_TYPE char6
#undef VALUE_TYPE_IS_PYOBJECT
#define TEST_VALUE(K, T) memcmp(K,T,6)
#define DECREF_VALUE(k)
#define INCREF_VALUE(k)
#define COPY_VALUE(V, E) (memcpy(V, E, 6))
#define COPY_VALUE_TO_OBJECT(O, K) O=PyString_FromStringAndSize((const char*)K,6)
#define COPY_VALUE_FROM_ARG(TARGET, ARG, STATUS) \
if ((PyString_Check(ARG) && PyString_GET_SIZE(ARG)==6)) \
memcpy(TARGET, PyString_AS_STRING(ARG), 6); else { \
PyErr_SetString(PyExc_TypeError, "expected six-character string key"); \
(STATUS)=0; }
#define NORMALIZE_VALUE(V, MIN)
#include "Python.h"
static PyObject *bucket_toString(PyObject *self);
static PyObject *bucket_fromString(PyObject *self, PyObject *state);
#define EXTRA_BUCKET_METHODS \
{"toString", (PyCFunction) bucket_toString, METH_NOARGS, \
"toString() -- Return the state as a string"}, \
{"fromString", (PyCFunction) bucket_fromString, METH_O, \
"fromString(s) -- Set the state of the object from a string"}, \
#include "BTreeModuleTemplate.c"
static PyObject *
bucket_toString(PyObject *oself)
{
Bucket *self = (Bucket *)oself;
PyObject *items = NULL;
int len;
PER_USE_OR_RETURN(self, NULL);
len = self->len;
items = PyString_FromStringAndSize(NULL, len*8);
if (items == NULL)
goto err;
memcpy(PyString_AS_STRING(items), self->keys, len*2);
memcpy(PyString_AS_STRING(items)+len*2, self->values, len*6);
PER_UNUSE(self);
return items;
err:
PER_UNUSE(self);
Py_XDECREF(items);
return NULL;
}
static PyObject *
bucket_fromString(PyObject *oself, PyObject *state)
{
Bucket *self = (Bucket *)oself;
int len;
KEY_TYPE *keys;
VALUE_TYPE *values;
len = PyString_Size(state);
if (len < 0)
return NULL;
if (len%8)
{
PyErr_SetString(PyExc_ValueError, "state string of wrong size");
return NULL;
}
len /= 8;
if (self->next) {
Py_DECREF(self->next);
self->next = NULL;
}
if (len > self->size) {
keys = BTree_Realloc(self->keys, sizeof(KEY_TYPE)*len);
if (keys == NULL)
return NULL;
values = BTree_Realloc(self->values, sizeof(VALUE_TYPE)*len);
if (values == NULL)
return NULL;
self->keys = keys;
self->values = values;
self->size = len;
}
memcpy(self->keys, PyString_AS_STRING(state), len*2);
memcpy(self->values, PyString_AS_STRING(state)+len*2, len*6);
self->len = len;
Py_INCREF(self);
return (PyObject *)self;
}
BTrees/check.py 0 → 100644
View file @ 703fc7e7
##############################################################################
#
# Copyright (c) 2003 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""
Utilities for working with BTrees (TreeSets, Buckets, and Sets) at a low
level.
The primary function is check(btree), which performs value-based consistency
checks of a kind btree._check() does not perform. See the function docstring
for details.
display(btree) displays the internal structure of a BTree (TreeSet, etc) to
stdout.
CAUTION: When a BTree node has only a single bucket child, it can be
impossible to get at the bucket from Python code (__getstate__() may squash
the bucket object out of existence, as a pickling storage optimization). In
such a case, the code here synthesizes a temporary bucket with the same keys
(and values, if the bucket is of a mapping type). This has no first-order
consequences, but can mislead if you pay close attention to reported object
addresses and/or object identity (the synthesized bucket has an address
that doesn't exist in the actual BTree).
"""
from BTrees.OOBTree import OOBTree, OOBucket, OOSet, OOTreeSet
from BTrees.OIBTree import OIBTree, OIBucket, OISet, OITreeSet
from BTrees.IOBTree import IOBTree, IOBucket, IOSet, IOTreeSet
from BTrees.IIBTree import IIBTree, IIBucket, IISet, IITreeSet
from BTrees.IFBTree import IFBTree, IFBucket, IFSet, IFTreeSet
from BTrees.OLBTree import OLBTree, OLBucket, OLSet, OLTreeSet
from BTrees.LOBTree import LOBTree, LOBucket, LOSet, LOTreeSet
from BTrees.LLBTree import LLBTree, LLBucket, LLSet, LLTreeSet
from BTrees.LFBTree import LFBTree, LFBucket, LFSet, LFTreeSet
from ZODB.utils import positive_id, oid_repr
TYPE_UNKNOWN, TYPE_BTREE, TYPE_BUCKET = range(3)
_type2kind = {}
for kv in ('OO',
'II', 'IO', 'OI', 'IF',
'LL', 'LO', 'OL', 'LF',
):
for name, kind in (
('BTree', (TYPE_BTREE, True)),
('Bucket', (TYPE_BUCKET, True)),
('TreeSet', (TYPE_BTREE, False)),
('Set', (TYPE_BUCKET, False)),
):
_type2kind[globals()[kv+name]] = kind
# Return pair
#
# TYPE_BTREE or TYPE_BUCKET, is_mapping
def classify(obj):
return _type2kind[type(obj)]
BTREE_EMPTY, BTREE_ONE, BTREE_NORMAL = range(3)
# If the BTree is empty, returns
#
# BTREE_EMPTY, [], []
#
# If the BTree has only one bucket, sometimes returns
#
# BTREE_ONE, bucket_state, None
#
# Else returns
#
# BTREE_NORMAL, list of keys, list of kids
#
# and the list of kids has one more entry than the list of keys.
#
# BTree.__getstate__() docs:
#
# For an empty BTree (self->len == 0), None.
#
# For a BTree with one child (self->len == 1), and that child is a bucket,
# and that bucket has a NULL oid, a one-tuple containing a one-tuple
# containing the bucket's state:
#
# (
# (
# child[0].__getstate__(),
# ),
# )
#
# Else a two-tuple. The first element is a tuple interleaving the BTree's
# keys and direct children, of size 2*self->len - 1 (key[0] is unused and
# is not saved). The second element is the firstbucket:
#
# (
# (child[0], key[1], child[1], key[2], child[2], ...,
# key[len-1], child[len-1]),
# self->firstbucket
# )
_btree2bucket = {}
for kv in ('OO',
'II', 'IO', 'OI', 'IF',
'LL', 'LO', 'OL', 'LF',
):
_btree2bucket[globals()[kv+'BTree']] = globals()[kv+'Bucket']
_btree2bucket[globals()[kv+'TreeSet']] = globals()[kv+'Set']
def crack_btree(t, is_mapping):
state = t.__getstate__()
if state is None:
return BTREE_EMPTY, [], []
assert isinstance(state, tuple)
if len(state) == 1:
state = state[0]
assert isinstance(state, tuple) and len(state) == 1
state = state[0]
return BTREE_ONE, state, None
assert len(state) == 2
data, firstbucket = state
n = len(data)
assert n & 1
kids = []
keys = []
i = 0
for x in data:
if i & 1:
keys.append(x)
else:
kids.append(x)
i += 1
return BTREE_NORMAL, keys, kids
# Returns
#
# keys, values # for a mapping; len(keys) == len(values) in this case
# or
# keys, [] # for a set
#
# bucket.__getstate__() docs:
#
# For a set bucket (self->values is NULL), a one-tuple or two-tuple. The
# first element is a tuple of keys, of length self->len. The second element
# is the next bucket, present if and only if next is non-NULL:
#
# (
# (keys[0], keys[1], ..., keys[len-1]),
# <self->next iff non-NULL>
# )
#
# For a mapping bucket (self->values is not NULL), a one-tuple or two-tuple.
# The first element is a tuple interleaving keys and values, of length
# 2 * self->len. The second element is the next bucket, present iff next is
# non-NULL:
#
# (
# (keys[0], values[0], keys[1], values[1], ...,
# keys[len-1], values[len-1]),
# <self->next iff non-NULL>
# )
def crack_bucket(b, is_mapping):
state = b.__getstate__()
assert isinstance(state, tuple)
assert 1 <= len(state) <= 2
data = state[0]
if not is_mapping:
return data, []
keys = []
values = []
n = len(data)
assert n & 1 == 0
i = 0
for x in data:
if i & 1:
values.append(x)
else:
keys.append(x)
i += 1
return keys, values
def type_and_adr(obj):
if hasattr(obj, '_p_oid'):
oid = oid_repr(obj._p_oid)
else:
oid = 'None'
return "%s (0x%x oid=%s)" % (type(obj).__name__, positive_id(obj), oid)
# Walker implements a depth-first search of a BTree (or TreeSet or Set or
# Bucket). Subclasses must implement the visit_btree() and visit_bucket()
# methods, and arrange to call the walk() method. walk() calls the
# visit_XYZ() methods once for each node in the tree, in depth-first
# left-to-right order.
class Walker:
def __init__(self, obj):
self.obj = obj
# obj is the BTree (BTree or TreeSet).
# path is a list of indices, from the root. For example, if a BTree node
# is child[5] of child[3] of the root BTree, [3, 5].
# parent is the parent BTree object, or None if this is the root BTree.
# is_mapping is True for a BTree and False for a TreeSet.
# keys is a list of the BTree's internal keys.
# kids is a list of the BTree's children.
# If the BTree is an empty root node, keys == kids == [].
# Else len(kids) == len(keys) + 1.
# lo and hi are slice bounds on the values the elements of keys *should*
# lie in (lo inclusive, hi exclusive). lo is None if there is no lower
# bound known, and hi is None if no upper bound is known.
def visit_btree(self, obj, path, parent, is_mapping,
keys, kids, lo, hi):
raise NotImplementedError
# obj is the bucket (Bucket or Set).
# path is a list of indices, from the root. For example, if a bucket
# node is child[5] of child[3] of the root BTree, [3, 5].
# parent is the parent BTree object.
# is_mapping is True for a Bucket and False for a Set.
# keys is a list of the bucket's keys.
# values is a list of the bucket's values.
# If is_mapping is false, values == []. Else len(keys) == len(values).
# lo and hi are slice bounds on the values the elements of keys *should*
# lie in (lo inclusive, hi exclusive). lo is None if there is no lower
# bound known, and hi is None if no upper bound is known.
def visit_bucket(self, obj, path, parent, is_mapping,
keys, values, lo, hi):
raise NotImplementedError
def walk(self):
obj = self.obj
path = []
stack = [(obj, path, None, None, None)]
while stack:
obj, path, parent, lo, hi = stack.pop()
kind, is_mapping = classify(obj)
if kind is TYPE_BTREE:
bkind, keys, kids = crack_btree(obj, is_mapping)
if bkind is BTREE_NORMAL:
# push the kids, in reverse order (so they're popped off
# the stack in forward order)
n = len(kids)
for i in range(len(kids)-1, -1, -1):
newlo, newhi = lo, hi
if i < n-1:
newhi = keys[i]
if i > 0:
newlo = keys[i-1]
stack.append((kids[i],
path + [i],
obj,
newlo,
newhi))
elif bkind is BTREE_EMPTY:
pass
else:
assert bkind is BTREE_ONE
# Yuck. There isn't a bucket object to pass on, as
# the bucket state is embedded directly in the BTree
# state. Synthesize a bucket.
assert kids is None # and "keys" is really the bucket
# state
bucket = _btree2bucket[type(obj)]()
bucket.__setstate__(keys)
stack.append((bucket,
path + [0],
obj,
lo,
hi))
keys = []
kids = [bucket]
self.visit_btree(obj,
path,
parent,
is_mapping,
keys,
kids,
lo,
hi)
else:
assert kind is TYPE_BUCKET
keys, values = crack_bucket(obj, is_mapping)
self.visit_bucket(obj,
path,
parent,
is_mapping,
keys,
values,
lo,
hi)
class Checker(Walker):
def __init__(self, obj):
Walker.__init__(self, obj)
self.errors = []
def check(self):
self.walk()
if self.errors:
s = "Errors found in %s:" % type_and_adr(self.obj)
self.errors.insert(0, s)
s = "\n".join(self.errors)
raise AssertionError(s)
def visit_btree(self, obj, path, parent, is_mapping,
keys, kids, lo, hi):
self.check_sorted(obj, path, keys, lo, hi)
def visit_bucket(self, obj, path, parent, is_mapping,
keys, values, lo, hi):
self.check_sorted(obj, path, keys, lo, hi)
def check_sorted(self, obj, path, keys, lo, hi):
i, n = 0, len(keys)
for x in keys:
if lo is not None and not lo <= x:
s = "key %r < lower bound %r at index %d" % (x, lo, i)
self.complain(s, obj, path)
if hi is not None and not x < hi:
s = "key %r >= upper bound %r at index %d" % (x, hi, i)
self.complain(s, obj, path)
if i < n-1 and not x < keys[i+1]:
s = "key %r at index %d >= key %r at index %d" % (
x, i, keys[i+1], i+1)
self.complain(s, obj, path)
i += 1
def complain(self, msg, obj, path):
s = "%s, in %s, path from root %s" % (
msg,
type_and_adr(obj),
".".join(map(str, path)))
self.errors.append(s)
class Printer(Walker):
def __init__(self, obj):
Walker.__init__(self, obj)
def display(self):
self.walk()
def visit_btree(self, obj, path, parent, is_mapping,
keys, kids, lo, hi):
indent = " " * len(path)
print "%s%s %s with %d children" % (
indent,
".".join(map(str, path)),
type_and_adr(obj),
len(kids))
indent += " "
n = len(keys)
for i in range(n):
print "%skey %d: %r" % (indent, i, keys[i])
def visit_bucket(self, obj, path, parent, is_mapping,
keys, values, lo, hi):
indent = " " * len(path)
print "%s%s %s with %d keys" % (
indent,
".".join(map(str, path)),
type_and_adr(obj),
len(keys))
indent += " "
n = len(keys)
for i in range(n):
print "%skey %d: %r" % (indent, i, keys[i]),
if is_mapping:
print "value %r" % (values[i],)
def check(btree):
"""Check internal value-based invariants in a BTree or TreeSet.
The btree._check() method checks internal C-level pointer consistency.
The check() function here checks value-based invariants: whether the
keys in leaf bucket and internal nodes are in strictly increasing order,
and whether they all lie in their expected range. The latter is a subtle
invariant that can't be checked locally -- it requires propagating
range info down from the root of the tree, and modifying it at each
level for each child.
Raises AssertionError if anything is wrong, with a string detail
explaining the problems. The entire tree is checked before
AssertionError is raised, and the string detail may be large (depending
on how much went wrong).
"""
Checker(btree).check()
def display(btree):
"Display the internal structure of a BTree, Bucket, TreeSet or Set."
Printer(btree).display()
BTrees/floatvaluemacros.h 0 → 100644
View file @ 703fc7e7
#define VALUEMACROS_H "$Id$\n"
#define VALUE_TYPE float
#undef VALUE_TYPE_IS_PYOBJECT
#define TEST_VALUE(K, T) (((K) < (T)) ? -1 : (((K) > (T)) ? 1: 0))
#define VALUE_SAME(VALUE, TARGET) ( (VALUE) == (TARGET) )
#define DECLARE_VALUE(NAME) VALUE_TYPE NAME
#define VALUE_PARSE "f"
#define DECREF_VALUE(k)
#define INCREF_VALUE(k)
#define COPY_VALUE(V, E) (V=(E))
#define COPY_VALUE_TO_OBJECT(O, K) O=PyFloat_FromDouble(K)
#define COPY_VALUE_FROM_ARG(TARGET, ARG, STATUS) \
if (PyFloat_Check(ARG)) TARGET = (float)PyFloat_AsDouble(ARG); \
else if (PyInt_Check(ARG)) TARGET = (float)PyInt_AsLong(ARG); \
else { \
PyErr_SetString(PyExc_TypeError, "expected float or int value"); \
(STATUS)=0; (TARGET)=0; }
#define NORMALIZE_VALUE(V, MIN) ((MIN) > 0) ? ((V)/=(MIN)) : 0
#define MERGE_DEFAULT 1.0f
#define MERGE(O1, w1, O2, w2) ((O1)*(w1)+(O2)*(w2))
#define MERGE_WEIGHT(O, w) ((O)*(w))
BTrees/fsBTree.py 0 → 100644
View file @ 703fc7e7
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
# fsBTrees are data structures used for ZODB FileStorage. They are not
# expected to be "public" excpect to FileStorage.
# hack to overcome dynamic-linking headache.
from _fsBTree import *
BTrees/intkeymacros.h 0 → 100644
View file @ 703fc7e7
#define KEYMACROS_H "$Id$\n"
#ifdef ZODB_64BIT_INTS
/* PY_LONG_LONG as key */
#define NEED_LONG_LONG_SUPPORT
#define KEY_TYPE PY_LONG_LONG
#define KEY_CHECK longlong_check
#define COPY_KEY_TO_OBJECT(O, K) O=longlong_as_object(K)
#define COPY_KEY_FROM_ARG(TARGET, ARG, STATUS) \
if (PyInt_Check(ARG)) TARGET=PyInt_AS_LONG(ARG); else \
if (longlong_check(ARG)) TARGET=PyLong_AsLongLong(ARG); else \
if (PyLong_Check(ARG)) { \
PyErr_SetString(PyExc_ValueError, "long integer out of range"); \
(STATUS)=0; (TARGET)=0; } \
else { \
PyErr_SetString(PyExc_TypeError, "expected integer key"); \
(STATUS)=0; (TARGET)=0; }
#else
/* C int as key */
#define KEY_TYPE int
#define KEY_CHECK PyInt_Check
#define COPY_KEY_TO_OBJECT(O, K) O=PyInt_FromLong(K)
#define COPY_KEY_FROM_ARG(TARGET, ARG, STATUS) \
if (PyInt_Check(ARG)) { \
long vcopy = PyInt_AS_LONG(ARG); \
if ((int)vcopy != vcopy) { \
PyErr_SetString(PyExc_TypeError, "integer out of range"); \
(STATUS)=0; (TARGET)=0; \
} \
else TARGET = vcopy; \
} else { \
PyErr_SetString(PyExc_TypeError, "expected integer key"); \
(STATUS)=0; (TARGET)=0; }
#endif
#undef KEY_TYPE_IS_PYOBJECT
#define TEST_KEY_SET_OR(V, K, T) if ( ( (V) = (((K) < (T)) ? -1 : (((K) > (T)) ? 1: 0)) ) , 0 )
#define DECREF_KEY(KEY)
#define INCREF_KEY(k)
#define COPY_KEY(KEY, E) (KEY=(E))
#define MULTI_INT_UNION 1
BTrees/intvaluemacros.h 0 → 100644
View file @ 703fc7e7
#define VALUEMACROS_H "$Id$\n"
#ifdef ZODB_64BIT_INTS
#define NEED_LONG_LONG_SUPPORT
#define VALUE_TYPE PY_LONG_LONG
#define VALUE_PARSE "L"
#define COPY_VALUE_TO_OBJECT(O, K) O=longlong_as_object(K)
#define COPY_VALUE_FROM_ARG(TARGET, ARG, STATUS) \
if (PyInt_Check(ARG)) TARGET=PyInt_AS_LONG(ARG); else \
if (longlong_check(ARG)) TARGET=PyLong_AsLongLong(ARG); else \
if (PyLong_Check(ARG)) { \
PyErr_SetString(PyExc_ValueError, "long integer out of range"); \
(STATUS)=0; (TARGET)=0; } \
else { \
PyErr_SetString(PyExc_TypeError, "expected integer value"); \
(STATUS)=0; (TARGET)=0; }
#else
#define VALUE_TYPE int
#define VALUE_PARSE "i"
#define COPY_VALUE_TO_OBJECT(O, K) O=PyInt_FromLong(K)
#define COPY_VALUE_FROM_ARG(TARGET, ARG, STATUS) \
if (PyInt_Check(ARG)) { \
long vcopy = PyInt_AS_LONG(ARG); \
if ((int)vcopy != vcopy) { \
PyErr_SetString(PyExc_TypeError, "integer out of range"); \
(STATUS)=0; (TARGET)=0; \
} \
else TARGET = vcopy; \
} else { \
PyErr_SetString(PyExc_TypeError, "expected integer key"); \
(STATUS)=0; (TARGET)=0; }
#endif
#undef VALUE_TYPE_IS_PYOBJECT
#define TEST_VALUE(K, T) (((K) < (T)) ? -1 : (((K) > (T)) ? 1: 0))
#define VALUE_SAME(VALUE, TARGET) ( (VALUE) == (TARGET) )
#define DECLARE_VALUE(NAME) VALUE_TYPE NAME
#define DECREF_VALUE(k)
#define INCREF_VALUE(k)
#define COPY_VALUE(V, E) (V=(E))
#define NORMALIZE_VALUE(V, MIN) ((MIN) > 0) ? ((V)/=(MIN)) : 0
#define MERGE_DEFAULT 1
#define MERGE(O1, w1, O2, w2) ((O1)*(w1)+(O2)*(w2))
#define MERGE_WEIGHT(O, w) ((O)*(w))
BTrees/objectkeymacros.h 0 → 100644
View file @ 703fc7e7
#define KEYMACROS_H "$Id$\n"
#define KEY_TYPE PyObject *
#define KEY_TYPE_IS_PYOBJECT
#include "Python.h"
static PyObject *object_;
static int
check_argument_cmp(PyObject *arg)
{
/* printf("check cmp %p %p %p %p\n", */
/* arg->ob_type->tp_richcompare, */
/* ((PyTypeObject *)object_)->ob_type->tp_richcompare, */
/* arg->ob_type->tp_compare, */
/* ((PyTypeObject *)object_)->ob_type->tp_compare); */
if (arg->ob_type->tp_richcompare == NULL
&&
#if PY_MAJOR_VERSION==2 && PY_MINOR_VERSION < 6
arg->ob_type->tp_compare == NULL
#else
arg->ob_type->tp_compare ==
((PyTypeObject *)object_)->ob_type->tp_compare
#endif
)
{
PyErr_SetString(PyExc_TypeError, "Object has default comparison");
return 0;
}
return 1;
}
#define TEST_KEY_SET_OR(V, KEY, TARGET) if ( ( (V) = PyObject_Compare((KEY),(TARGET)) ), PyErr_Occurred() )
#define INCREF_KEY(k) Py_INCREF(k)
#define DECREF_KEY(KEY) Py_DECREF(KEY)
#define COPY_KEY(KEY, E) KEY=(E)
#define COPY_KEY_TO_OBJECT(O, K) O=(K); Py_INCREF(O)
#define COPY_KEY_FROM_ARG(TARGET, ARG, S) \
TARGET=(ARG); \
(S) = check_argument_cmp(ARG);
BTrees/objectvaluemacros.h 0 → 100644
View file @ 703fc7e7
#define VALUEMACROS_H "$Id$\n"
#define VALUE_TYPE PyObject *
#define VALUE_TYPE_IS_PYOBJECT
#define TEST_VALUE(VALUE, TARGET) PyObject_Compare((VALUE),(TARGET))
#define DECLARE_VALUE(NAME) VALUE_TYPE NAME
#define INCREF_VALUE(k) Py_INCREF(k)
#define DECREF_VALUE(k) Py_DECREF(k)
#define COPY_VALUE(k,e) k=(e)
#define COPY_VALUE_TO_OBJECT(O, K) O=(K); Py_INCREF(O)
#define COPY_VALUE_FROM_ARG(TARGET, ARG, S) TARGET=(ARG)
#define NORMALIZE_VALUE(V, MIN) Py_INCREF(V)
BTrees/py24compat.h 0 → 100644
View file @ 703fc7e7
/* Backport type definitions from Python 2.5's object.h */
#ifndef BTREE_PY24COMPATH_H
#define BTREE_PY24COMPAT_H
#if PY_VERSION_HEX < 0x02050000
typedef Py_ssize_t (*lenfunc)(PyObject *);
typedef PyObject *(*ssizeargfunc)(PyObject *, Py_ssize_t);
typedef PyObject *(*ssizessizeargfunc)(PyObject *, Py_ssize_t, Py_ssize_t);
typedef int(*ssizeobjargproc)(PyObject *, Py_ssize_t, PyObject *);
typedef int(*ssizessizeobjargproc)(PyObject *, Py_ssize_t, Py_ssize_t, PyObject *);
#endif /* PY_VERSION_HEX */
#endif /* BTREE_PY24COMPAT_H */
BTrees/sorters.c 0 → 100644
View file @ 703fc7e7
/*****************************************************************************
Copyright (c) 2002 Zope Foundation and Contributors.
All Rights Reserved.
This software is subject to the provisions of the Zope Public License,
Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
****************************************************************************/
/* Revision information: $Id$ */
/* The only routine here intended to be used outside the file is
size_t sort_int_nodups(int *p, size_t n)
Sort the array of n ints pointed at by p, in place, and also remove
duplicates. Return the number of unique elements remaining, which occupy
a contiguous and monotonically increasing slice of the array starting at p.
Example: If the input array is [3, 1, 2, 3, 1, 5, 2], sort_int_nodups
returns 4, and the first 4 elements of the array are changed to
[1, 2, 3, 5]. The content of the remaining array positions is not defined.
Notes:
+ This is specific to n-byte signed ints, with endianness natural to the
platform. `n` is determined based on ZODB_64BIT_INTS.
+ 4*n bytes of available heap memory are required for best speed
(8*n when ZODB_64BIT_INTS is defined).
*/
#include <stdlib.h>
#include <stddef.h>
#include <memory.h>
#include <string.h>
#include <assert.h>
/* The type of array elements to be sorted. Most of the routines don't
care about the type, and will work fine for any scalar C type (provided
they're recompiled with element_type appropriately redefined). However,
the radix sort has to know everything about the type's internal
representation.
*/
typedef KEY_TYPE element_type;
/* The radixsort is faster than the quicksort for large arrays, but radixsort
has high fixed overhead, making it a poor choice for small arrays. The
crossover point isn't critical, and is sensitive to things like compiler
and machine cache structure, so don't worry much about this.
*/
#define QUICKSORT_BEATS_RADIXSORT 800U
/* In turn, the quicksort backs off to an insertion sort for very small
slices. MAX_INSERTION is the largest slice quicksort leaves entirely to
insertion. Because this version of quicksort uses a median-of-3 rule for
selecting a pivot, MAX_INSERTION must be at least 2 (so that quicksort
has at least 3 values to look at in a slice). Again, the exact value here
isn't critical.
*/
#define MAX_INSERTION 25U
#if MAX_INSERTION < 2U
# error "MAX_INSERTION must be >= 2"
#endif
/* LSB-first radix sort of the n elements in 'in'.
'work' is work storage at least as large as 'in'. Depending on how many
swaps are done internally, the final result may come back in 'in' or 'work';
and that pointer is returned.
radixsort_int is specific to signed n-byte ints, with natural machine
endianness. `n` is determined based on ZODB_64BIT_INTS.
*/
static element_type*
radixsort_int(element_type *in, element_type *work, size_t n)
{
/* count[i][j] is the number of input elements that have byte value j
in byte position i, where byte position 0 is the LSB. Note that
holding i fixed, the sum of count[i][j] over all j in range(256)
is n.
*/
#ifdef ZODB_64BIT_INTS
size_t count[8][256];
#else
size_t count[4][256];
#endif
size_t i;
int offset, offsetinc;
/* Which byte position are we working on now? 0=LSB, 1, 2, ... */
int bytenum;
#ifdef ZODB_64BIT_INTS
assert(sizeof(element_type) == 8);
#else
assert(sizeof(element_type) == 4);
#endif
assert(in);
assert(work);
/* Compute all of count in one pass. */
memset(count, 0, sizeof(count));
for (i = 0; i < n; ++i) {
element_type const x = in[i];
++count[0][(x ) & 0xff];
++count[1][(x >> 8) & 0xff];
++count[2][(x >> 16) & 0xff];
++count[3][(x >> 24) & 0xff];
#ifdef ZODB_64BIT_INTS
++count[4][(x >> 32) & 0xff];
++count[5][(x >> 40) & 0xff];
++count[6][(x >> 48) & 0xff];
++count[7][(x >> 56) & 0xff];
#endif
}
/* For p an element_type* cast to char*, offset is how much farther we
have to go to get to the LSB of the element; this is 0 for little-
endian boxes and sizeof(element_type)-1 for big-endian.
offsetinc is 1 or -1, respectively, telling us which direction to go
from p+offset to get to the element's more-significant bytes.
*/
{
element_type one = 1;
if (*(char*)&one) {
/* Little endian. */
offset = 0;
offsetinc = 1;
}
else {
/* Big endian. */
offset = sizeof(element_type) - 1;
offsetinc = -1;
}
}
/* The radix sort. */
for (bytenum = 0;
bytenum < sizeof(element_type);
++bytenum, offset += offsetinc) {
/* Do a stable distribution sort on byte position bytenum,
from in to work. index[i] tells us the work index at which
to store the next in element with byte value i. pinbyte
points to the correct byte in the input array.
*/
size_t index[256];
unsigned char* pinbyte;
size_t total = 0;
size_t *pcount = count[bytenum];
/* Compute the correct output starting index for each possible
byte value.
*/
if (bytenum < sizeof(element_type) - 1) {
for (i = 0; i < 256; ++i) {
const size_t icount = pcount[i];
index[i] = total;
total += icount;
if (icount == n)
break;
}
if (i < 256) {
/* All bytes in the current position have value
i, so there's nothing to do on this pass.
*/
continue;
}
}
else {
/* The MSB of signed ints needs to be distributed
differently than the other bytes, in order
0x80, 0x81, ... 0xff, 0x00, 0x01, ... 0x7f
*/
for (i = 128; i < 256; ++i) {
const size_t icount = pcount[i];
index[i] = total;
total += icount;
if (icount == n)
break;
}
if (i < 256)
continue;
for (i = 0; i < 128; ++i) {
const size_t icount = pcount[i];
index[i] = total;
total += icount;
if (icount == n)
break;
}
if (i < 128)
continue;
}
assert(total == n);
/* Distribute the elements according to byte value. Note that
this is where most of the time is spent.
Note: The loop is unrolled 4x by hand, for speed. This
may be a pessimization someday, but was a significant win
on my MSVC 6.0 timing tests.
*/
pinbyte = (unsigned char *)in + offset;
i = 0;
/* Reduce number of elements to copy to a multiple of 4. */
while ((n - i) & 0x3) {
unsigned char byte = *pinbyte;
work[index[byte]++] = in[i];
++i;
pinbyte += sizeof(element_type);
}
for (; i < n; i += 4, pinbyte += 4 * sizeof(element_type)) {
unsigned char byte1 = *(pinbyte );
unsigned char byte2 = *(pinbyte + sizeof(element_type));
unsigned char byte3 = *(pinbyte + 2 * sizeof(element_type));
unsigned char byte4 = *(pinbyte + 3 * sizeof(element_type));
element_type in1 = in[i ];
element_type in2 = in[i+1];
element_type in3 = in[i+2];
element_type in4 = in[i+3];
work[index[byte1]++] = in1;
work[index[byte2]++] = in2;
work[index[byte3]++] = in3;
work[index[byte4]++] = in4;
}
/* Swap in and work (just a pointer swap). */
{
element_type *temp = in;
in = work;
work = temp;
}
}
return in;
}
/* Remove duplicates from sorted array in, storing exactly one of each distinct
element value into sorted array out. It's OK (and expected!) for in == out,
but otherwise the n elements beginning at in must not overlap with the n
beginning at out.
Return the number of elements in out.
*/
static size_t
uniq(element_type *out, element_type *in, size_t n)
{
size_t i;
element_type lastelt;
element_type *pout;
assert(out);
assert(in);
if (n == 0)
return 0;
/* i <- first index in 'in' that contains a duplicate.
in[0], in[1], ... in[i-1] are unique, but in[i-1] == in[i].
Set i to n if everything is unique.
*/
for (i = 1; i < n; ++i) {
if (in[i-1] == in[i])
break;
}
/* in[:i] is unique; copy to out[:i] if needed. */
assert(i > 0);
if (in != out)
memcpy(out, in, i * sizeof(element_type));
pout = out + i;
lastelt = in[i-1]; /* safe even when i == n */
for (++i; i < n; ++i) {
element_type elt = in[i];
if (elt != lastelt)
*pout++ = lastelt = elt;
}
return pout - out;
}
#if 0
/* insertionsort is no longer referenced directly, but I'd like to keep
* the code here just in case.
*/
/* Straight insertion sort of the n elements starting at 'in'. */
static void
insertionsort(element_type *in, size_t n)
{
element_type *p, *q;
element_type minimum; /* smallest seen so far */
element_type *plimit = in + n;
assert(in);
if (n < 2)
return;
minimum = *in;
for (p = in+1; p < plimit; ++p) {
/* *in <= *(in+1) <= ... <= *(p-1). Slide *p into place. */
element_type thiselt = *p;
if (thiselt < minimum) {
/* This is a new minimum. This saves p-in compares
when it happens, but should happen so rarely that
it's not worth checking for its own sake: the
point is that the far more popular 'else' branch can
exploit that thiselt is *not* the smallest so far.
*/
memmove(in+1, in, (p - in) * sizeof(*in));
*in = minimum = thiselt;
}
else {
/* thiselt >= minimum, so the loop will find a q
with *q <= thiselt. This saves testing q >= in
on each trip. It's such a simple loop that saving
a per-trip test is a major speed win.
*/
for (q = p-1; *q > thiselt; --q)
*(q+1) = *q;
*(q+1) = thiselt;
}
}
}
#endif
/* The maximum number of elements in the pending-work stack quicksort
maintains. The maximum stack depth is approximately log2(n), so
arrays of size up to approximately MAX_INSERTION * 2**STACKSIZE can be
sorted. The memory burden for the stack is small, so better safe than
sorry.
*/
#define STACKSIZE 60
/* A _stacknode remembers a contiguous slice of an array that needs to sorted.
lo must be <= hi, and, unlike Python array slices, this includes both ends.
*/
struct _stacknode {
element_type *lo;
element_type *hi;
};
static void
quicksort(element_type *plo, size_t n)
{
element_type *phi;
/* Swap two array elements. */
element_type _temp;
#define SWAP(P, Q) (_temp = *(P), *(P) = *(Q), *(Q) = _temp)
/* Stack of pending array slices to be sorted. */
struct _stacknode stack[STACKSIZE];
struct _stacknode *stackfree = stack; /* available stack slot */
/* Push an array slice on the pending-work stack. */
#define PUSH(PLO, PHI) \
do { \
assert(stackfree - stack < STACKSIZE); \
assert((PLO) <= (PHI)); \
stackfree->lo = (PLO); \
stackfree->hi = (PHI); \
++stackfree; \
} while(0)
assert(plo);
phi = plo + n - 1;
for (;;) {
element_type pivot;
element_type *pi, *pj;
assert(plo <= phi);
n = phi - plo + 1;
if (n <= MAX_INSERTION) {
/* Do a small insertion sort. Contra Knuth, we do
this now instead of waiting until the end, because
this little slice is likely still in cache now.
*/
element_type *p, *q;
element_type minimum = *plo;
for (p = plo+1; p <= phi; ++p) {
/* *plo <= *(plo+1) <= ... <= *(p-1).
Slide *p into place. */
element_type thiselt = *p;
if (thiselt < minimum) {
/* New minimum. */
memmove(plo+1,
plo,
(p - plo) * sizeof(*p));
*plo = minimum = thiselt;
}
else {
/* thiselt >= minimum, so the loop will
find a q with *q <= thiselt.
*/
for (q = p-1; *q > thiselt; --q)
*(q+1) = *q;
*(q+1) = thiselt;
}
}
/* Pop another slice off the stack. */
if (stack == stackfree)
break; /* no more slices -- we're done */
--stackfree;
plo = stackfree->lo;
phi = stackfree->hi;
continue;
}
/* Parition the slice.
For pivot, take the median of the leftmost, rightmost, and
middle elements. First sort those three; then the median
is the middle one. For technical reasons, the middle
element is swapped to plo+1 first (see Knuth Vol 3 Ed 2
section 5.2.2 exercise 55 -- reverse-sorted arrays can
take quadratic time otherwise!).
*/
{
element_type *plop1 = plo + 1;
element_type *pmid = plo + (n >> 1);
assert(plo < pmid && pmid < phi);
SWAP(plop1, pmid);
/* Sort plo, plop1, phi. */
/* Smaller of rightmost two -> middle. */
if (*plop1 > *phi)
SWAP(plop1, phi);
/* Smallest of all -> left; if plo is already the
smallest, the sort is complete.
*/
if (*plo > *plop1) {
SWAP(plo, plop1);
/* Largest of all -> right. */
if (*plop1 > *phi)
SWAP(plop1, phi);
}
pivot = *plop1;
pi = plop1;
}
assert(*plo <= pivot);
assert(*pi == pivot);
assert(*phi >= pivot);
pj = phi;
/* Partition wrt pivot. This is the time-critical part, and
nearly every decision in the routine aims at making this
loop as fast as possible -- even small points like
arranging that all loop tests can be done correctly at the
bottoms of loops instead of the tops, and that pointers can
be derefenced directly as-is (without fiddly +1 or -1).
The aim is to make the C here so simple that a compiler
has a good shot at doing as well as hand-crafted assembler.
*/
for (;;) {
/* Invariants:
1. pi < pj.
2. All elements at plo, plo+1 .. pi are <= pivot.
3. All elements at pj, pj+1 .. phi are >= pivot.
4. There is an element >= pivot to the right of pi.
5. There is an element <= pivot to the left of pj.
Note that #4 and #5 save us from needing to check
that the pointers stay in bounds.
*/
assert(pi < pj);
do { ++pi; } while (*pi < pivot);
assert(pi <= pj);
do { --pj; } while (*pj > pivot);
assert(pj >= pi - 1);
if (pi < pj)
SWAP(pi, pj);
else
break;
}
assert(plo+1 < pi && pi <= phi);
assert(plo < pj && pj < phi);
assert(*pi >= pivot);
assert( (pi == pj && *pj == pivot) ||
(pj + 1 == pi && *pj <= pivot) );
/* Swap pivot into its final position, pj. */
assert(plo[1] == pivot);
plo[1] = *pj;
*pj = pivot;
/* Subfiles are from plo to pj-1 inclusive, and pj+1 to phi
inclusive. Push the larger one, and loop back to do the
smaller one directly.
*/
if (pj - plo >= phi - pj) {
PUSH(plo, pj-1);
plo = pj+1;
}
else {
PUSH(pj+1, phi);
phi = pj-1;
}
}
#undef PUSH
#undef SWAP
}
/* Sort p and remove duplicates, as fast as we can. */
static size_t
sort_int_nodups(KEY_TYPE *p, size_t n)
{
size_t nunique;
element_type *work;
assert(sizeof(KEY_TYPE) == sizeof(element_type));
assert(p);
/* Use quicksort if the array is small, OR if malloc can't find
enough temp memory for radixsort.
*/
work = NULL;
if (n > QUICKSORT_BEATS_RADIXSORT)
work = (element_type *)malloc(n * sizeof(element_type));
if (work) {
element_type *out = radixsort_int(p, work, n);
nunique = uniq(p, out, n);
free(work);
}
else {
quicksort(p, n);
nunique = uniq(p, p, n);
}
return nunique;
}
BTrees/tests/__init__.py 0 → 100644
View file @ 703fc7e7
# If tests is a package, debugging is a bit easier.
BTrees/tests/testBTrees.py 0 → 100644
View file @ 703fc7e7
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import gc
import pickle
import random
import StringIO
from unittest import TestCase, TestSuite, TextTestRunner, makeSuite
from types import ClassType
import zope.interface.verify
from BTrees.OOBTree import OOBTree, OOBucket, OOSet, OOTreeSet
from BTrees.IOBTree import IOBTree, IOBucket, IOSet, IOTreeSet
from BTrees.IIBTree import IIBTree, IIBucket, IISet, IITreeSet
from BTrees.IFBTree import IFBTree, IFBucket, IFSet, IFTreeSet
from BTrees.OIBTree import OIBTree, OIBucket, OISet, OITreeSet
from BTrees.LOBTree import LOBTree, LOBucket, LOSet, LOTreeSet
from BTrees.LLBTree import LLBTree, LLBucket, LLSet, LLTreeSet
from BTrees.LFBTree import LFBTree, LFBucket, LFSet, LFTreeSet
from BTrees.OLBTree import OLBTree, OLBucket, OLSet, OLTreeSet
import BTrees
from BTrees.IIBTree import using64bits
from BTrees.check import check
import transaction
from ZODB import DB
from ZODB.MappingStorage import MappingStorage
class Base(TestCase):
""" Tests common to all types: sets, buckets, and BTrees """
db = None
def setUp(self):
self.t = self.t_class()
def tearDown(self):
if self.db is not None:
self.db.close()
self.t = None
del self.t
def _getRoot(self):
if self.db is None:
# Unclear: On the next line, the ZODB4 flavor of this routine
# [asses a cache_size argument:
# self.db = DB(MappingStorage(), cache_size=1)
# If that's done here, though, testLoadAndStore() and
# testGhostUnghost() both nail the CPU and seemingly
# never finish.
self.db = DB(MappingStorage())
return self.db.open().root()
def _closeRoot(self, root):
root._p_jar.close()
def testLoadAndStore(self):
for i in 0, 10, 1000:
t = self.t.__class__()
self._populate(t, i)
root = None
root = self._getRoot()
root[i] = t
transaction.commit()
root2 = self._getRoot()
if hasattr(t, 'items'):
self.assertEqual(list(root2[i].items()) , list(t.items()))
else:
self.assertEqual(list(root2[i].keys()) , list(t.keys()))
self._closeRoot(root)
self._closeRoot(root2)
def testSetstateArgumentChecking(self):
try: self.t.__class__().__setstate__(('',))
except TypeError, v:
self.assertEqual(str(v), 'tuple required for first state element')
else:
raise AssertionError("Expected exception")
def testGhostUnghost(self):
for i in 0, 10, 1000:
t = self.t.__class__()
self._populate(t, i)
root = self._getRoot()
root[i] = t
transaction.commit()
root2 = self._getRoot()
root2[i]._p_deactivate()
transaction.commit()
if hasattr(t, 'items'):
self.assertEqual(list(root2[i].items()) , list(t.items()))
else:
self.assertEqual(list(root2[i].keys()) , list(t.keys()))
self._closeRoot(root)
self._closeRoot(root2)
def testSimpleExclusiveKeyRange(self):
t = self.t.__class__()
self.assertEqual(list(t.keys()), [])
self.assertEqual(list(t.keys(excludemin=True)), [])
self.assertEqual(list(t.keys(excludemax=True)), [])
self.assertEqual(list(t.keys(excludemin=True, excludemax=True)), [])
self._populate(t, 1)
self.assertEqual(list(t.keys()), [0])
self.assertEqual(list(t.keys(excludemin=True)), [])
self.assertEqual(list(t.keys(excludemax=True)), [])
self.assertEqual(list(t.keys(excludemin=True, excludemax=True)), [])
t.clear()
self._populate(t, 2)
self.assertEqual(list(t.keys()), [0, 1])
self.assertEqual(list(t.keys(excludemin=True)), [1])
self.assertEqual(list(t.keys(excludemax=True)), [0])
self.assertEqual(list(t.keys(excludemin=True, excludemax=True)), [])
t.clear()
self._populate(t, 3)
self.assertEqual(list(t.keys()), [0, 1, 2])
self.assertEqual(list(t.keys(excludemin=True)), [1, 2])
self.assertEqual(list(t.keys(excludemax=True)), [0, 1])
self.assertEqual(list(t.keys(excludemin=True, excludemax=True)), [1])
self.assertEqual(list(t.keys(-1, 3, excludemin=True, excludemax=True)),
[0, 1, 2])
self.assertEqual(list(t.keys(0, 3, excludemin=True, excludemax=True)),
[1, 2])
self.assertEqual(list(t.keys(-1, 2, excludemin=True, excludemax=True)),
[0, 1])
self.assertEqual(list(t.keys(0, 2, excludemin=True, excludemax=True)),
[1])
def testUpdatesDoReadChecksOnInternalNodes(self):
t = self.t
if not hasattr(t, '_firstbucket'):
return
self._populate(t, 1000)
store = MappingStorage()
db = DB(store)
conn = db.open()
conn.root.t = t
transaction.commit()
read = []
def readCurrent(ob):
read.append(ob)
conn.__class__.readCurrent(conn, ob)
return 1
conn.readCurrent = readCurrent
try:
add = t.add
remove = t.remove
except AttributeError:
def add(i):
t[i] = i
def remove(i):
del t[i]
# Modifying a thing
remove(100)
self.assert_(t in read)
del read[:]
add(100)
self.assert_(t in read)
del read[:]
transaction.abort()
conn.cacheMinimize()
list(t)
self.assert_(100 in t)
self.assert_(not read)
class MappingBase(Base):
""" Tests common to mappings (buckets, btrees) """
def _populate(self, t, l):
# Make some data
for i in range(l): t[i]=i
def testRepr(self):
# test the repr because buckets have a complex repr implementation
# internally the cutoff from a stack allocated buffer to a heap
# allocated buffer is 10000.
for i in range(1000):
self.t[i] = i
r = repr(self.t)
# Make sure the repr is 10000 bytes long for a bucket.
# But since the test is also run for btrees, skip the length
# check if the repr starts with '<'
if not r.startswith('<'):
self.assert_(len(r) > 10000)
def testGetItemFails(self):
self.assertRaises(KeyError, self._getitemfail)
def _getitemfail(self):
return self.t[1]
def testGetReturnsDefault(self):
self.assertEqual(self.t.get(1) , None)
self.assertEqual(self.t.get(1, 'foo') , 'foo')
def testSetItemGetItemWorks(self):
self.t[1] = 1
a = self.t[1]
self.assertEqual(a , 1, `a`)
def testReplaceWorks(self):
self.t[1] = 1
self.assertEqual(self.t[1] , 1, self.t[1])
self.t[1] = 2
self.assertEqual(self.t[1] , 2, self.t[1])
def testLen(self):
added = {}
r = range(1000)
for x in r:
k = random.choice(r)
self.t[k] = x
added[k] = x
addl = added.keys()
self.assertEqual(len(self.t) , len(addl), len(self.t))
def testHasKeyWorks(self):
self.t[1] = 1
self.assert_(self.t.has_key(1))
self.assert_(1 in self.t)
self.assert_(0 not in self.t)
self.assert_(2 not in self.t)
def testValuesWorks(self):
for x in range(100):
self.t[x] = x*x
v = self.t.values()
for i in range(100):
self.assertEqual(v[i], i*i)
self.assertRaises(IndexError, lambda: v[i+1])
i = 0
for value in self.t.itervalues():
self.assertEqual(value, i*i)
i += 1
def testValuesWorks1(self):
for x in range(100):
self.t[99-x] = x
for x in range(40):
lst = list(self.t.values(0+x,99-x))
lst.sort()
self.assertEqual(lst,range(0+x,99-x+1))
lst = list(self.t.values(max=99-x, min=0+x))
lst.sort()
self.assertEqual(lst,range(0+x,99-x+1))
def testValuesNegativeIndex(self):
L = [-3, 6, -11, 4]
for i in L:
self.t[i] = i
L.sort()
vals = self.t.values()
for i in range(-1, -5, -1):
self.assertEqual(vals[i], L[i])
self.assertRaises(IndexError, lambda: vals[-5])
def testKeysWorks(self):
for x in range(100):
self.t[x] = x
v = self.t.keys()
i = 0
for x in v:
self.assertEqual(x,i)
i = i + 1
self.assertRaises(IndexError, lambda: v[i])
for x in range(40):
lst = self.t.keys(0+x,99-x)
self.assertEqual(list(lst), range(0+x, 99-x+1))
lst = self.t.keys(max=99-x, min=0+x)
self.assertEqual(list(lst), range(0+x, 99-x+1))
self.assertEqual(len(v), 100)
def testKeysNegativeIndex(self):
L = [-3, 6, -11, 4]
for i in L:
self.t[i] = i
L.sort()
keys = self.t.keys()
for i in range(-1, -5, -1):
self.assertEqual(keys[i], L[i])
self.assertRaises(IndexError, lambda: keys[-5])
def testItemsWorks(self):
for x in range(100):
self.t[x] = 2*x
v = self.t.items()
i = 0
for x in v:
self.assertEqual(x[0], i)
self.assertEqual(x[1], 2*i)
i += 1
self.assertRaises(IndexError, lambda: v[i+1])
i = 0
for x in self.t.iteritems():
self.assertEqual(x, (i, 2*i))
i += 1
items = list(self.t.items(min=12, max=20))
self.assertEqual(items, zip(range(12, 21), range(24, 43, 2)))
items = list(self.t.iteritems(min=12, max=20))
self.assertEqual(items, zip(range(12, 21), range(24, 43, 2)))
def testItemsNegativeIndex(self):
L = [-3, 6, -11, 4]
for i in L:
self.t[i] = i
L.sort()
items = self.t.items()
for i in range(-1, -5, -1):
self.assertEqual(items[i], (L[i], L[i]))
self.assertRaises(IndexError, lambda: items[-5])
def testDeleteInvalidKeyRaisesKeyError(self):
self.assertRaises(KeyError, self._deletefail)
def _deletefail(self):
del self.t[1]
def testMaxKeyMinKey(self):
self.t[7] = 6
self.t[3] = 10
self.t[8] = 12
self.t[1] = 100
self.t[5] = 200
self.t[10] = 500
self.t[6] = 99
self.t[4] = 150
del self.t[7]
t = self.t
self.assertEqual(t.maxKey(), 10)
self.assertEqual(t.maxKey(6), 6)
self.assertEqual(t.maxKey(9), 8)
self.assertEqual(t.minKey(), 1)
self.assertEqual(t.minKey(3), 3)
self.assertEqual(t.minKey(9), 10)
try:
t.maxKey(t.minKey() - 1)
except ValueError, err:
self.assertEqual(str(err), "no key satisfies the conditions")
else:
self.fail("expected ValueError")
try:
t.minKey(t.maxKey() + 1)
except ValueError, err:
self.assertEqual(str(err), "no key satisfies the conditions")
else:
self.fail("expected ValueError")
def testClear(self):
r = range(100)
for x in r:
rnd = random.choice(r)
self.t[rnd] = 0
self.t.clear()
diff = lsubtract(list(self.t.keys()), [])
self.assertEqual(diff, [])
def testUpdate(self):
d={}
l=[]
for i in range(10000):
k=random.randrange(-2000, 2001)
d[k]=i
l.append((k, i))
items=d.items()
items.sort()
self.t.update(d)
self.assertEqual(list(self.t.items()), items)
self.t.clear()
self.assertEqual(list(self.t.items()), [])
self.t.update(l)
self.assertEqual(list(self.t.items()), items)
# Before ZODB 3.4.2, update/construction from PersistentMapping failed.
def testUpdateFromPersistentMapping(self):
from persistent.mapping import PersistentMapping
pm = PersistentMapping({1: 2})
self.t.update(pm)
self.assertEqual(list(self.t.items()), [(1, 2)])
# Construction goes thru the same internals as .update().
t = self.t.__class__(pm)
self.assertEqual(list(t.items()), [(1, 2)])
def testEmptyRangeSearches(self):
t = self.t
t.update([(1,1), (5,5), (9,9)])
self.assertEqual(list(t.keys(-6,-4)), [], list(t.keys(-6,-4)))
self.assertEqual(list(t.keys(2,4)), [], list(t.keys(2,4)))
self.assertEqual(list(t.keys(6,8)), [], list(t.keys(6,8)))
self.assertEqual(list(t.keys(10,12)), [], list(t.keys(10,12)))
self.assertEqual(list(t.keys(9, 1)), [], list(t.keys(9, 1)))
# For IITreeSets, this one was returning 31 for len(keys), and
# list(keys) produced a list with 100 elements.
t.clear()
t.update(zip(range(300), range(300)))
keys = t.keys(200, 50)
self.assertEqual(len(keys), 0)
self.assertEqual(list(keys), [])
self.assertEqual(list(t.iterkeys(200, 50)), [])
keys = t.keys(max=50, min=200)
self.assertEqual(len(keys), 0)
self.assertEqual(list(keys), [])
self.assertEqual(list(t.iterkeys(max=50, min=200)), [])
def testSlicing(self):
# Test that slicing of .keys()/.values()/.items() works exactly the
# same way as slicing a Python list with the same contents.
# This tests fixes to several bugs in this area, starting with
# http://collector.zope.org/Zope/419,
# "BTreeItems slice contains 1 too many elements".
t = self.t
for n in range(10):
t.clear()
self.assertEqual(len(t), 0)
keys = []
values = []
items = []
for key in range(n):
value = -2 * key
t[key] = value
keys.append(key)
values.append(value)
items.append((key, value))
self.assertEqual(len(t), n)
kslice = t.keys()
vslice = t.values()
islice = t.items()
self.assertEqual(len(kslice), n)
self.assertEqual(len(vslice), n)
self.assertEqual(len(islice), n)
# Test whole-structure slices.
x = kslice[:]
self.assertEqual(list(x), keys[:])
x = vslice[:]
self.assertEqual(list(x), values[:])
x = islice[:]
self.assertEqual(list(x), items[:])
for lo in range(-2*n, 2*n+1):
# Test one-sided slices.
x = kslice[:lo]
self.assertEqual(list(x), keys[:lo])
x = kslice[lo:]
self.assertEqual(list(x), keys[lo:])
x = vslice[:lo]
self.assertEqual(list(x), values[:lo])
x = vslice[lo:]
self.assertEqual(list(x), values[lo:])
x = islice[:lo]
self.assertEqual(list(x), items[:lo])
x = islice[lo:]
self.assertEqual(list(x), items[lo:])
for hi in range(-2*n, 2*n+1):
# Test two-sided slices.
x = kslice[lo:hi]
self.assertEqual(list(x), keys[lo:hi])
x = vslice[lo:hi]
self.assertEqual(list(x), values[lo:hi])
x = islice[lo:hi]
self.assertEqual(list(x), items[lo:hi])
# The specific test case from Zope collector 419.
t.clear()
for i in xrange(100):
t[i] = 1
tslice = t.items()[20:80]
self.assertEqual(len(tslice), 60)
self.assertEqual(list(tslice), zip(range(20, 80), [1]*60))
def testIterators(self):
t = self.t
for keys in [], [-2], [1, 4], range(-170, 2000, 6):
t.clear()
for k in keys:
t[k] = -3 * k
self.assertEqual(list(t), keys)
x = []
for k in t:
x.append(k)
self.assertEqual(x, keys)
it = iter(t)
self.assert_(it is iter(it))
x = []
try:
while 1:
x.append(it.next())
except StopIteration:
pass
self.assertEqual(x, keys)
self.assertEqual(list(t.iterkeys()), keys)
self.assertEqual(list(t.itervalues()), list(t.values()))
self.assertEqual(list(t.iteritems()), list(t.items()))
def testRangedIterators(self):
t = self.t
for keys in [], [-2], [1, 4], range(-170, 2000, 13):
t.clear()
values = []
for k in keys:
value = -3 * k
t[k] = value
values.append(value)
items = zip(keys, values)
self.assertEqual(list(t.iterkeys()), keys)
self.assertEqual(list(t.itervalues()), values)
self.assertEqual(list(t.iteritems()), items)
if not keys:
continue
min_mid_max = (keys[0], keys[len(keys) >> 1], keys[-1])
for key1 in min_mid_max:
for lo in range(key1 - 1, key1 + 2):
# Test one-sided range iterators.
goodkeys = [k for k in keys if lo <= k]
got = t.iterkeys(lo)
self.assertEqual(goodkeys, list(got))
goodvalues = [t[k] for k in goodkeys]
got = t.itervalues(lo)
self.assertEqual(goodvalues, list(got))
gooditems = zip(goodkeys, goodvalues)
got = t.iteritems(lo)
self.assertEqual(gooditems, list(got))
for key2 in min_mid_max:
for hi in range(key2 - 1, key2 + 2):
goodkeys = [k for k in keys if lo <= k <= hi]
got = t.iterkeys(min=lo, max=hi)
self.assertEqual(goodkeys, list(got))
goodvalues = [t[k] for k in goodkeys]
got = t.itervalues(lo, max=hi)
self.assertEqual(goodvalues, list(got))
gooditems = zip(goodkeys, goodvalues)
got = t.iteritems(max=hi, min=lo)
self.assertEqual(gooditems, list(got))
def testBadUpdateTupleSize(self):
# This one silently ignored the excess in Zope3.
try:
self.t.update([(1, 2, 3)])
except TypeError:
pass
else:
self.fail("update() with 3-tuple didn't complain")
# This one dumped core in Zope3.
try:
self.t.update([(1,)])
except TypeError:
pass
else:
self.fail("update() with 1-tuple didn't complain")
# This one should simply succeed.
self.t.update([(1, 2)])
self.assertEqual(list(self.t.items()), [(1, 2)])
def testSimpleExclusivRanges(self):
def identity(x):
return x
def dup(x):
return [(y, y) for y in x]
for methodname, f in (("keys", identity),
("values", identity),
("items", dup),
("iterkeys", identity),
("itervalues", identity),
("iteritems", dup)):
t = self.t.__class__()
meth = getattr(t, methodname, None)
if meth is None:
continue
self.assertEqual(list(meth()), [])
self.assertEqual(list(meth(excludemin=True)), [])
self.assertEqual(list(meth(excludemax=True)), [])
self.assertEqual(list(meth(excludemin=True, excludemax=True)), [])
self._populate(t, 1)
self.assertEqual(list(meth()), f([0]))
self.assertEqual(list(meth(excludemin=True)), [])
self.assertEqual(list(meth(excludemax=True)), [])
self.assertEqual(list(meth(excludemin=True, excludemax=True)), [])
t.clear()
self._populate(t, 2)
self.assertEqual(list(meth()), f([0, 1]))
self.assertEqual(list(meth(excludemin=True)), f([1]))
self.assertEqual(list(meth(excludemax=True)), f([0]))
self.assertEqual(list(meth(excludemin=True, excludemax=True)), [])
t.clear()
self._populate(t, 3)
self.assertEqual(list(meth()), f([0, 1, 2]))
self.assertEqual(list(meth(excludemin=True)), f([1, 2]))
self.assertEqual(list(meth(excludemax=True)), f([0, 1]))
self.assertEqual(list(meth(excludemin=True, excludemax=True)),
f([1]))
self.assertEqual(list(meth(-1, 3, excludemin=True,
excludemax=True)),
f([0, 1, 2]))
self.assertEqual(list(meth(0, 3, excludemin=True,
excludemax=True)),
f([1, 2]))
self.assertEqual(list(meth(-1, 2, excludemin=True,
excludemax=True)),
f([0, 1]))
self.assertEqual(list(meth(0, 2, excludemin=True,
excludemax=True)),
f([1]))
def testSetdefault(self):
t = self.t
self.assertEqual(t.setdefault(1, 2), 2)
# That should also have associated 1 with 2 in the tree.
self.assert_(1 in t)
self.assertEqual(t[1], 2)
# And trying to change it again should have no effect.
self.assertEqual(t.setdefault(1, 666), 2)
self.assertEqual(t[1], 2)
# Not enough arguments.
self.assertRaises(TypeError, t.setdefault)
self.assertRaises(TypeError, t.setdefault, 1)
# Too many arguments.
self.assertRaises(TypeError, t.setdefault, 1, 2, 3)
def testPop(self):
t = self.t
# Empty container.
# If no default given, raises KeyError.
self.assertRaises(KeyError, t.pop, 1)
# But if default given, returns that instead.
self.assertEqual(t.pop(1, 42), 42)
t[1] = 3
# KeyError when key is not in container and default is not passed.
self.assertRaises(KeyError, t.pop, 5)
self.assertEqual(list(t.items()), [(1, 3)])
# If key is in container, returns the value and deletes the key.
self.assertEqual(t.pop(1), 3)
self.assertEqual(len(t), 0)
# If key is present, return value bypassing default.
t[1] = 3
self.assertEqual(t.pop(1, 7), 3)
self.assertEqual(len(t), 0)
# Pop only one item.
t[1] = 3
t[2] = 4
self.assertEqual(len(t), 2)
self.assertEqual(t.pop(1), 3)
self.assertEqual(len(t), 1)
self.assertEqual(t[2], 4)
self.assertEqual(t.pop(1, 3), 3)
# Too few arguments.
self.assertRaises(TypeError, t.pop)
# Too many arguments.
self.assertRaises(TypeError, t.pop, 1, 2, 3)
class NormalSetTests(Base):
""" Test common to all set types """
def _populate(self, t, l):
# Make some data
t.update(range(l))
def testInsertReturnsValue(self):
t = self.t
self.assertEqual(t.insert(5) , 1)
self.assertEqual(t.add(4) , 1)
def testDuplicateInsert(self):
t = self.t
t.insert(5)
self.assertEqual(t.insert(5) , 0)
self.assertEqual(t.add(5) , 0)
def testInsert(self):
t = self.t
t.insert(1)
self.assert_(t.has_key(1))
self.assert_(1 in t)
self.assert_(2 not in t)
def testBigInsert(self):
t = self.t
r = xrange(10000)
for x in r:
t.insert(x)
for x in r:
self.assert_(t.has_key(x))
self.assert_(x in t)
def testRemoveSucceeds(self):
t = self.t
r = xrange(10000)
for x in r: t.insert(x)
for x in r: t.remove(x)
def testRemoveFails(self):
self.assertRaises(KeyError, self._removenonexistent)
def _removenonexistent(self):
self.t.remove(1)
def testHasKeyFails(self):
t = self.t
self.assert_(not t.has_key(1))
self.assert_(1 not in t)
def testKeys(self):
t = self.t
r = xrange(1000)
for x in r:
t.insert(x)
diff = lsubtract(t.keys(), r)
self.assertEqual(diff, [])
def testClear(self):
t = self.t
r = xrange(1000)
for x in r: t.insert(x)
t.clear()
diff = lsubtract(t.keys(), [])
self.assertEqual(diff , [], diff)
def testMaxKeyMinKey(self):
t = self.t
t.insert(1)
t.insert(2)
t.insert(3)
t.insert(8)
t.insert(5)
t.insert(10)
t.insert(6)
t.insert(4)
self.assertEqual(t.maxKey() , 10)
self.assertEqual(t.maxKey(6) , 6)
self.assertEqual(t.maxKey(9) , 8)
self.assertEqual(t.minKey() , 1)
self.assertEqual(t.minKey(3) , 3)
self.assertEqual(t.minKey(9) , 10)
self.assert_(t.minKey() in t)
self.assert_(t.minKey()-1 not in t)
self.assert_(t.maxKey() in t)
self.assert_(t.maxKey()+1 not in t)
try:
t.maxKey(t.minKey() - 1)
except ValueError, err:
self.assertEqual(str(err), "no key satisfies the conditions")
else:
self.fail("expected ValueError")
try:
t.minKey(t.maxKey() + 1)
except ValueError, err:
self.assertEqual(str(err), "no key satisfies the conditions")
else:
self.fail("expected ValueError")
def testUpdate(self):
d={}
l=[]
for i in range(10000):
k=random.randrange(-2000, 2001)
d[k]=i
l.append(k)
items = d.keys()
items.sort()
self.t.update(l)
self.assertEqual(list(self.t.keys()), items)
def testEmptyRangeSearches(self):
t = self.t
t.update([1, 5, 9])
self.assertEqual(list(t.keys(-6,-4)), [], list(t.keys(-6,-4)))
self.assertEqual(list(t.keys(2,4)), [], list(t.keys(2,4)))
self.assertEqual(list(t.keys(6,8)), [], list(t.keys(6,8)))
self.assertEqual(list(t.keys(10,12)), [], list(t.keys(10,12)))
self.assertEqual(list(t.keys(9,1)), [], list(t.keys(9,1)))
# For IITreeSets, this one was returning 31 for len(keys), and
# list(keys) produced a list with 100 elements.
t.clear()
t.update(range(300))
keys = t.keys(200, 50)
self.assertEqual(len(keys), 0)
self.assertEqual(list(keys), [])
keys = t.keys(max=50, min=200)
self.assertEqual(len(keys), 0)
self.assertEqual(list(keys), [])
def testSlicing(self):
# Test that slicing of .keys() works exactly the same way as slicing
# a Python list with the same contents.
t = self.t
for n in range(10):
t.clear()
self.assertEqual(len(t), 0)
keys = range(10*n, 11*n)
t.update(keys)
self.assertEqual(len(t), n)
kslice = t.keys()
self.assertEqual(len(kslice), n)
# Test whole-structure slices.
x = kslice[:]
self.assertEqual(list(x), keys[:])
for lo in range(-2*n, 2*n+1):
# Test one-sided slices.
x = kslice[:lo]
self.assertEqual(list(x), keys[:lo])
x = kslice[lo:]
self.assertEqual(list(x), keys[lo:])
for hi in range(-2*n, 2*n+1):
# Test two-sided slices.
x = kslice[lo:hi]
self.assertEqual(list(x), keys[lo:hi])
def testIterator(self):
t = self.t
for keys in [], [-2], [1, 4], range(-170, 2000, 6):
t.clear()
t.update(keys)
self.assertEqual(list(t), keys)
x = []
for k in t:
x.append(k)
self.assertEqual(x, keys)
it = iter(t)
self.assert_(it is iter(it))
x = []
try:
while 1:
x.append(it.next())
except StopIteration:
pass
self.assertEqual(x, keys)
class ExtendedSetTests(NormalSetTests):
def testLen(self):
t = self.t
r = xrange(10000)
for x in r: t.insert(x)
self.assertEqual(len(t) , 10000, len(t))
def testGetItem(self):
t = self.t
r = xrange(10000)
for x in r: t.insert(x)
for x in r:
self.assertEqual(t[x] , x)
class BTreeTests(MappingBase):
""" Tests common to all BTrees """
def tearDown(self):
self.t._check()
check(self.t)
MappingBase.tearDown(self)
def testDeleteNoChildrenWorks(self):
self.t[5] = 6
self.t[2] = 10
self.t[6] = 12
self.t[1] = 100
self.t[3] = 200
self.t[10] = 500
self.t[4] = 99
del self.t[4]
diff = lsubtract(self.t.keys(), [1,2,3,5,6,10])
self.assertEqual(diff , [], diff)
def testDeleteOneChildWorks(self):
self.t[5] = 6
self.t[2] = 10
self.t[6] = 12
self.t[1] = 100
self.t[3] = 200
self.t[10] = 500
self.t[4] = 99
del self.t[3]
diff = lsubtract(self.t.keys(), [1,2,4,5,6,10])
self.assertEqual(diff , [], diff)
def testDeleteTwoChildrenNoInorderSuccessorWorks(self):
self.t[5] = 6
self.t[2] = 10
self.t[6] = 12
self.t[1] = 100
self.t[3] = 200
self.t[10] = 500
self.t[4] = 99
del self.t[2]
diff = lsubtract(self.t.keys(), [1,3,4,5,6,10])
self.assertEqual(diff , [], diff)
def testDeleteTwoChildrenInorderSuccessorWorks(self):
# 7, 3, 8, 1, 5, 10, 6, 4 -- del 3
self.t[7] = 6
self.t[3] = 10
self.t[8] = 12
self.t[1] = 100
self.t[5] = 200
self.t[10] = 500
self.t[6] = 99
self.t[4] = 150
del self.t[3]
diff = lsubtract(self.t.keys(), [1,4,5,6,7,8,10])
self.assertEqual(diff , [], diff)
def testDeleteRootWorks(self):
# 7, 3, 8, 1, 5, 10, 6, 4 -- del 7
self.t[7] = 6
self.t[3] = 10
self.t[8] = 12
self.t[1] = 100
self.t[5] = 200
self.t[10] = 500
self.t[6] = 99
self.t[4] = 150
del self.t[7]
diff = lsubtract(self.t.keys(), [1,3,4,5,6,8,10])
self.assertEqual(diff , [], diff)
def testRandomNonOverlappingInserts(self):
added = {}
r = range(100)
for x in r:
k = random.choice(r)
if not added.has_key(k):
self.t[k] = x
added[k] = 1
addl = added.keys()
addl.sort()
diff = lsubtract(list(self.t.keys()), addl)
self.assertEqual(diff , [], (diff, addl, list(self.t.keys())))
def testRandomOverlappingInserts(self):
added = {}
r = range(100)
for x in r:
k = random.choice(r)
self.t[k] = x
added[k] = 1
addl = added.keys()
addl.sort()
diff = lsubtract(self.t.keys(), addl)
self.assertEqual(diff , [], diff)
def testRandomDeletes(self):
r = range(1000)
added = []
for x in r:
k = random.choice(r)
self.t[k] = x
added.append(k)
deleted = []
for x in r:
k = random.choice(r)
if self.t.has_key(k):
self.assert_(k in self.t)
del self.t[k]
deleted.append(k)
if self.t.has_key(k):
self.fail( "had problems deleting %s" % k )
badones = []
for x in deleted:
if self.t.has_key(x):
badones.append(x)
self.assertEqual(badones , [], (badones, added, deleted))
def testTargetedDeletes(self):
r = range(1000)
for x in r:
k = random.choice(r)
self.t[k] = x
for x in r:
try:
del self.t[x]
except KeyError:
pass
self.assertEqual(realseq(self.t.keys()) , [], realseq(self.t.keys()))
def testPathologicalRightBranching(self):
r = range(1000)
for x in r:
self.t[x] = 1
self.assertEqual(realseq(self.t.keys()) , r, realseq(self.t.keys()))
for x in r:
del self.t[x]
self.assertEqual(realseq(self.t.keys()) , [], realseq(self.t.keys()))
def testPathologicalLeftBranching(self):
r = range(1000)
revr = r[:]
revr.reverse()
for x in revr:
self.t[x] = 1
self.assertEqual(realseq(self.t.keys()) , r, realseq(self.t.keys()))
for x in revr:
del self.t[x]
self.assertEqual(realseq(self.t.keys()) , [], realseq(self.t.keys()))
def testSuccessorChildParentRewriteExerciseCase(self):
add_order = [
85, 73, 165, 273, 215, 142, 233, 67, 86, 166, 235, 225, 255,
73, 175, 171, 285, 162, 108, 28, 283, 258, 232, 199, 260,
298, 275, 44, 261, 291, 4, 181, 285, 289, 216, 212, 129,
243, 97, 48, 48, 159, 22, 285, 92, 110, 27, 55, 202, 294,
113, 251, 193, 290, 55, 58, 239, 71, 4, 75, 129, 91, 111,
271, 101, 289, 194, 218, 77, 142, 94, 100, 115, 101, 226,
17, 94, 56, 18, 163, 93, 199, 286, 213, 126, 240, 245, 190,
195, 204, 100, 199, 161, 292, 202, 48, 165, 6, 173, 40, 218,
271, 228, 7, 166, 173, 138, 93, 22, 140, 41, 234, 17, 249,
215, 12, 292, 246, 272, 260, 140, 58, 2, 91, 246, 189, 116,
72, 259, 34, 120, 263, 168, 298, 118, 18, 28, 299, 192, 252,
112, 60, 277, 273, 286, 15, 263, 141, 241, 172, 255, 52, 89,
127, 119, 255, 184, 213, 44, 116, 231, 173, 298, 178, 196,
89, 184, 289, 98, 216, 115, 35, 132, 278, 238, 20, 241, 128,
179, 159, 107, 206, 194, 31, 260, 122, 56, 144, 118, 283,
183, 215, 214, 87, 33, 205, 183, 212, 221, 216, 296, 40,
108, 45, 188, 139, 38, 256, 276, 114, 270, 112, 214, 191,
147, 111, 299, 107, 101, 43, 84, 127, 67, 205, 251, 38, 91,
297, 26, 165, 187, 19, 6, 73, 4, 176, 195, 90, 71, 30, 82,
139, 210, 8, 41, 253, 127, 190, 102, 280, 26, 233, 32, 257,
194, 263, 203, 190, 111, 218, 199, 29, 81, 207, 18, 180,
157, 172, 192, 135, 163, 275, 74, 296, 298, 265, 105, 191,
282, 277, 83, 188, 144, 259, 6, 173, 81, 107, 292, 231,
129, 65, 161, 113, 103, 136, 255, 285, 289, 1
]
delete_order = [
276, 273, 12, 275, 2, 286, 127, 83, 92, 33, 101, 195,
299, 191, 22, 232, 291, 226, 110, 94, 257, 233, 215, 184,
35, 178, 18, 74, 296, 210, 298, 81, 265, 175, 116, 261,
212, 277, 260, 234, 6, 129, 31, 4, 235, 249, 34, 289, 105,
259, 91, 93, 119, 7, 183, 240, 41, 253, 290, 136, 75, 292,
67, 112, 111, 256, 163, 38, 126, 139, 98, 56, 282, 60, 26,
55, 245, 225, 32, 52, 40, 271, 29, 252, 239, 89, 87, 205,
213, 180, 97, 108, 120, 218, 44, 187, 196, 251, 202, 203,
172, 28, 188, 77, 90, 199, 297, 282, 141, 100, 161, 216,
73, 19, 17, 189, 30, 258
]
for x in add_order:
self.t[x] = 1
for x in delete_order:
try: del self.t[x]
except KeyError:
if self.t.has_key(x):
self.assertEqual(1,2,"failed to delete %s" % x)
def testRangeSearchAfterSequentialInsert(self):
r = range(100)
for x in r:
self.t[x] = 0
diff = lsubtract(list(self.t.keys(0, 100)), r)
self.assertEqual(diff , [], diff)
def testRangeSearchAfterRandomInsert(self):
r = range(100)
a = {}
for x in r:
rnd = random.choice(r)
self.t[rnd] = 0
a[rnd] = 0
diff = lsubtract(list(self.t.keys(0, 100)), a.keys())
self.assertEqual(diff , [], diff)
def testPathologicalRangeSearch(self):
t = self.t
# Build a 2-level tree with at least two buckets.
for i in range(200):
t[i] = i
items, dummy = t.__getstate__()
self.assert_(len(items) > 2) # at least two buckets and a key
# All values in the first bucket are < firstkey. All in the
# second bucket are >= firstkey, and firstkey is the first key in
# the second bucket.
firstkey = items[1]
therange = t.keys(-1, firstkey)
self.assertEqual(len(therange), firstkey + 1)
self.assertEqual(list(therange), range(firstkey + 1))
# Now for the tricky part. If we delete firstkey, the second bucket
# loses its smallest key, but firstkey remains in the BTree node.
# If we then do a high-end range search on firstkey, the BTree node
# directs us to look in the second bucket, but there's no longer any
# key <= firstkey in that bucket. The correct answer points to the
# end of the *first* bucket. The algorithm has to be smart enough
# to "go backwards" in the BTree then; if it doesn't, it will
# erroneously claim that the range is empty.
del t[firstkey]
therange = t.keys(min=-1, max=firstkey)
self.assertEqual(len(therange), firstkey)
self.assertEqual(list(therange), range(firstkey))
def testInsertMethod(self):
t = self.t
t[0] = 1
self.assertEqual(t.insert(0, 1) , 0)
self.assertEqual(t.insert(1, 1) , 1)
self.assertEqual(lsubtract(list(t.keys()), [0,1]) , [])
def testDamagedIterator(self):
# A cute one from Steve Alexander. This caused the BTreeItems
# object to go insane, accessing memory beyond the allocated part
# of the bucket. If it fails, the symptom is either a C-level
# assertion error (if the BTree code was compiled without NDEBUG),
# or most likely a segfault (if the BTree code was compiled with
# NDEBUG).
t = self.t.__class__()
self._populate(t, 10)
# In order for this to fail, it's important that k be a "lazy"
# iterator, referring to the BTree by indirect position (index)
# instead of a fully materialized list. Then the position can
# end up pointing into trash memory, if the bucket pointed to
# shrinks.
k = t.keys()
for dummy in range(20):
try:
del t[k[0]]
except RuntimeError, detail:
self.assertEqual(str(detail), "the bucket being iterated "
"changed size")
break
LARGEST_32_BITS = 2147483647
SMALLEST_32_BITS = -LARGEST_32_BITS - 1
SMALLEST_POSITIVE_33_BITS = LARGEST_32_BITS + 1
LARGEST_NEGATIVE_33_BITS = SMALLEST_32_BITS - 1
LARGEST_64_BITS = 0x7fffffffffffffff
SMALLEST_64_BITS = -LARGEST_64_BITS - 1
SMALLEST_POSITIVE_65_BITS = LARGEST_64_BITS + 1
LARGEST_NEGATIVE_65_BITS = SMALLEST_64_BITS - 1
class TestLongIntSupport:
def getTwoValues(self):
"""Return two distinct values; these must compare as un-equal.
These values must be usable as values.
"""
return object(), object()
def getTwoKeys(self):
"""Return two distinct values, these must compare as un-equal.
These values must be usable as keys.
"""
return 0, 1
def _set_value(self, key, value):
self.t[key] = value
class TestLongIntKeys(TestLongIntSupport):
def testLongIntKeysWork(self):
o1, o2 = self.getTwoValues()
assert o1 != o2
# Test some small key values first:
self.t[0L] = o1
self.assertEqual(self.t[0], o1)
self.t[0] = o2
self.assertEqual(self.t[0L], o2)
self.assertEqual(list(self.t.keys()), [0])
# Test some large key values too:
k1 = SMALLEST_POSITIVE_33_BITS
k2 = LARGEST_64_BITS
k3 = SMALLEST_64_BITS
self.t[k1] = o1
self.t[k2] = o2
self.t[k3] = o1
self.assertEqual(self.t[k1], o1)
self.assertEqual(self.t[k2], o2)
self.assertEqual(self.t[k3], o1)
self.assertEqual(list(self.t.keys()), [k3, 0, k1, k2])
def testLongIntKeysOutOfRange(self):
o1, o2 = self.getTwoValues()
self.assertRaises(
ValueError,
self._set_value, SMALLEST_POSITIVE_65_BITS, o1)
self.assertRaises(
ValueError,
self._set_value, LARGEST_NEGATIVE_65_BITS, o1)
class TestLongIntValues(TestLongIntSupport):
def testLongIntValuesWork(self):
keys = list(self.getTwoKeys())
keys.sort()
k1, k2 = keys
assert k1 != k2
# This is the smallest positive integer that requires 33 bits:
v1 = SMALLEST_POSITIVE_33_BITS
v2 = v1 + 1
self.t[k1] = v1
self.t[k2] = v2
self.assertEqual(self.t[k1], v1)
self.assertEqual(self.t[k2], v2)
self.assertEqual(list(self.t.values()), [v1, v2])
def testLongIntValuesOutOfRange(self):
k1, k2 = self.getTwoKeys()
self.assertRaises(
ValueError,
self._set_value, k1, SMALLEST_POSITIVE_65_BITS)
self.assertRaises(
ValueError,
self._set_value, k1, LARGEST_NEGATIVE_65_BITS)
if not using64bits:
# We're not using 64-bit ints in this build, so we don't expect
# the long-integer tests to pass.
class TestLongIntKeys:
pass
class TestLongIntValues:
pass
# tests of various type errors
class TypeTest(TestCase):
def testBadTypeRaises(self):
self.assertRaises(TypeError, self._stringraises)
self.assertRaises(TypeError, self._floatraises)
self.assertRaises(TypeError, self._noneraises)
class TestIOBTrees(TypeTest):
def setUp(self):
self.t = IOBTree()
def _stringraises(self):
self.t['c'] = 1
def _floatraises(self):
self.t[2.5] = 1
def _noneraises(self):
self.t[None] = 1
class TestOIBTrees(TypeTest):
def setUp(self):
self.t = OIBTree()
def _stringraises(self):
self.t[1] = 'c'
def _floatraises(self):
self.t[1] = 1.4
def _noneraises(self):
self.t[1] = None
def testEmptyFirstBucketReportedByGuido(self):
b = self.t
for i in xrange(29972): # reduce to 29971 and it works
b[i] = i
for i in xrange(30): # reduce to 29 and it works
del b[i]
b[i+40000] = i
self.assertEqual(b.keys()[0], 30)
class TestIIBTrees(TestCase):
def setUp(self):
self.t = IIBTree()
def testNonIntegerKeyRaises(self):
self.assertRaises(TypeError, self._stringraiseskey)
self.assertRaises(TypeError, self._floatraiseskey)
self.assertRaises(TypeError, self._noneraiseskey)
def testNonIntegerValueRaises(self):
self.assertRaises(TypeError, self._stringraisesvalue)
self.assertRaises(TypeError, self._floatraisesvalue)
self.assertRaises(TypeError, self._noneraisesvalue)
def _stringraiseskey(self):
self.t['c'] = 1
def _floatraiseskey(self):
self.t[2.5] = 1
def _noneraiseskey(self):
self.t[None] = 1
def _stringraisesvalue(self):
self.t[1] = 'c'
def _floatraisesvalue(self):
self.t[1] = 1.4
def _noneraisesvalue(self):
self.t[1] = None
class TestIFBTrees(TestCase):
def setUp(self):
self.t = IFBTree()
def testNonIntegerKeyRaises(self):
self.assertRaises(TypeError, self._stringraiseskey)
self.assertRaises(TypeError, self._floatraiseskey)
self.assertRaises(TypeError, self._noneraiseskey)
def testNonNumericValueRaises(self):
self.assertRaises(TypeError, self._stringraisesvalue)
self.assertRaises(TypeError, self._noneraisesvalue)
self.t[1] = 1
self.t[1] = 1.0
def _stringraiseskey(self):
self.t['c'] = 1
def _floatraiseskey(self):
self.t[2.5] = 1
def _noneraiseskey(self):
self.t[None] = 1
def _stringraisesvalue(self):
self.t[1] = 'c'
def _floatraisesvalue(self):
self.t[1] = 1.4
def _noneraisesvalue(self):
self.t[1] = None
class TestI_Sets(TestCase):
def testBadBadKeyAfterFirst(self):
self.assertRaises(TypeError, self.t.__class__, [1, ''])
self.assertRaises(TypeError, self.t.update, [1, ''])
del self.t
def testNonIntegerInsertRaises(self):
self.assertRaises(TypeError,self._insertstringraises)
self.assertRaises(TypeError,self._insertfloatraises)
self.assertRaises(TypeError,self._insertnoneraises)
def _insertstringraises(self):
self.t.insert('a')
def _insertfloatraises(self):
self.t.insert(1.4)
def _insertnoneraises(self):
self.t.insert(None)
class TestIOSets(TestI_Sets):
def setUp(self):
self.t = IOSet()
class TestIOTreeSets(TestI_Sets):
def setUp(self):
self.t = IOTreeSet()
class TestIISets(TestI_Sets):
def setUp(self):
self.t = IISet()
class TestIITreeSets(TestI_Sets):
def setUp(self):
self.t = IITreeSet()
class TestLOSets(TestI_Sets):
def setUp(self):
self.t = LOSet()
class TestLOTreeSets(TestI_Sets):
def setUp(self):
self.t = LOTreeSet()
class TestLLSets(TestI_Sets):
def setUp(self):
self.t = LLSet()
class TestLLTreeSets(TestI_Sets):
def setUp(self):
self.t = LLTreeSet()
class DegenerateBTree(TestCase):
# Build a degenerate tree (set). Boxes are BTree nodes. There are
# 5 leaf buckets, each containing a single int. Keys in the BTree
# nodes don't appear in the buckets. Seven BTree nodes are purely
# indirection nodes (no keys). Buckets aren't all at the same depth:
#
# +------------------------+
# | 4 |
# +------------------------+
# | |
# | v
# | +-+
# | | |
# | +-+
# | |
# v v
# +-------+ +-------------+
# | 2 | | 6 10 |
# +-------+ +-------------+
# | | | | |
# v v v v v
# +-+ +-+ +-+ +-+ +-+
# | | | | | | | | | |
# +-+ +-+ +-+ +-+ +-+
# | | | | |
# v v v v v
# 1 3 +-+ 7 11
# | |
# +-+
# |
# v
# 5
#
# This is nasty for many algorithms. Consider a high-end range search
# for 4. The BTree nodes direct it to the 5 bucket, but the correct
# answer is the 3 bucket, which requires going in a different direction
# at the very top node already. Consider a low-end range search for
# 9. The BTree nodes direct it to the 7 bucket, but the correct answer
# is the 11 bucket. This is also a nasty-case tree for deletions.
def _build_degenerate_tree(self):
# Build the buckets and chain them together.
bucket11 = IISet([11])
bucket7 = IISet()
bucket7.__setstate__(((7,), bucket11))
bucket5 = IISet()
bucket5.__setstate__(((5,), bucket7))
bucket3 = IISet()
bucket3.__setstate__(((3,), bucket5))
bucket1 = IISet()
bucket1.__setstate__(((1,), bucket3))
# Build the deepest layers of indirection nodes.
ts = IITreeSet
tree1 = ts()
tree1.__setstate__(((bucket1,), bucket1))
tree3 = ts()
tree3.__setstate__(((bucket3,), bucket3))
tree5lower = ts()
tree5lower.__setstate__(((bucket5,), bucket5))
tree5 = ts()
tree5.__setstate__(((tree5lower,), bucket5))
tree7 = ts()
tree7.__setstate__(((bucket7,), bucket7))
tree11 = ts()
tree11.__setstate__(((bucket11,), bucket11))
# Paste together the middle layers.
tree13 = ts()
tree13.__setstate__(((tree1, 2, tree3), bucket1))
tree5711lower = ts()
tree5711lower.__setstate__(((tree5, 6, tree7, 10, tree11), bucket5))
tree5711 = ts()
tree5711.__setstate__(((tree5711lower,), bucket5))
# One more.
t = ts()
t.__setstate__(((tree13, 4, tree5711), bucket1))
t._check()
check(t)
return t, [1, 3, 5, 7, 11]
def testBasicOps(self):
t, keys = self._build_degenerate_tree()
self.assertEqual(len(t), len(keys))
self.assertEqual(list(t.keys()), keys)
# has_key actually returns the depth of a bucket.
self.assertEqual(t.has_key(1), 4)
self.assertEqual(t.has_key(3), 4)
self.assertEqual(t.has_key(5), 6)
self.assertEqual(t.has_key(7), 5)
self.assertEqual(t.has_key(11), 5)
for i in 0, 2, 4, 6, 8, 9, 10, 12:
self.assert_(i not in t)
def _checkRanges(self, tree, keys):
self.assertEqual(len(tree), len(keys))
sorted_keys = keys[:]
sorted_keys.sort()
self.assertEqual(list(tree.keys()), sorted_keys)
for k in keys:
self.assert_(k in tree)
if keys:
lokey = sorted_keys[0]
hikey = sorted_keys[-1]
self.assertEqual(lokey, tree.minKey())
self.assertEqual(hikey, tree.maxKey())
else:
lokey = hikey = 42
# Try all range searches.
for lo in range(lokey - 1, hikey + 2):
for hi in range(lo - 1, hikey + 2):
for skipmin in False, True:
for skipmax in False, True:
wantlo, wanthi = lo, hi
if skipmin:
wantlo += 1
if skipmax:
wanthi -= 1
want = [k for k in keys if wantlo <= k <= wanthi]
got = list(tree.keys(lo, hi, skipmin, skipmax))
self.assertEqual(want, got)
def testRanges(self):
t, keys = self._build_degenerate_tree()
self._checkRanges(t, keys)
def testDeletes(self):
# Delete keys in all possible orders, checking each tree along
# the way.
# This is a tough test. Previous failure modes included:
# 1. A variety of assertion failures in _checkRanges.
# 2. Assorted "Invalid firstbucket pointer" failures at
# seemingly random times, coming out of the BTree destructor.
# 3. Under Python 2.3 CVS, some baffling
# RuntimeWarning: tp_compare didn't return -1 or -2 for exception
# warnings, possibly due to memory corruption after a BTree
# goes insane.
t, keys = self._build_degenerate_tree()
for oneperm in permutations(keys):
t, keys = self._build_degenerate_tree()
for key in oneperm:
t.remove(key)
keys.remove(key)
t._check()
check(t)
self._checkRanges(t, keys)
# We removed all the keys, so the tree should be empty now.
self.assertEqual(t.__getstate__(), None)
# A damaged tree may trigger an "invalid firstbucket pointer"
# failure at the time its destructor is invoked. Try to force
# that to happen now, so it doesn't look like a baffling failure
# at some unrelated line.
del t # trigger destructor
LP294788_ids = {}
class ToBeDeleted(object):
def __init__(self, id):
assert type(id) is int #we don't want to store any object ref here
self.id = id
global LP294788_ids
LP294788_ids[id] = 1
def __del__(self):
global LP294788_ids
LP294788_ids.pop(self.id, None)
def __cmp__(self, other):
return cmp(self.id, other.id)
def __hash__(self):
return hash(self.id)
class BugFixes(TestCase):
# Collector 1843. Error returns were effectively ignored in
# Bucket_rangeSearch(), leading to "delayed" errors, or worse.
def testFixed1843(self):
t = IISet()
t.insert(1)
# This one used to fail to raise the TypeError when it occurred.
self.assertRaises(TypeError, t.keys, "")
# This one used to segfault.
self.assertRaises(TypeError, t.keys, 0, "")
def test_LP294788(self):
# https://bugs.launchpad.net/bugs/294788
# BTree keeps some deleted objects referenced
# The logic here together with the ToBeDeleted class is that
# a separate reference dict is populated on object creation
# and removed in __del__
# That means what's left in the reference dict is never GC'ed
# therefore referenced somewhere
# To simulate real life, some random data is used to exercise the tree
t = OOBTree()
trandom = random.Random('OOBTree')
global LP294788_ids
# /// BTree keys are integers, value is an object
LP294788_ids = {}
ids = {}
for i in xrange(1024):
if trandom.random() > 0.1:
#add
id = None
while id is None or id in ids:
id = trandom.randint(0,1000000)
ids[id] = 1
t[id] = ToBeDeleted(id)
else:
#del
id = trandom.choice(ids.keys())
del t[id]
del ids[id]
ids = ids.keys()
trandom.shuffle(ids)
for id in ids:
del t[id]
ids = None
#to be on the safe side run a full GC
gc.collect()
#print LP294788_ids
self.assertEqual(len(t), 0)
self.assertEqual(len(LP294788_ids), 0)
# \\\
# /// BTree keys are integers, value is a tuple having an object
LP294788_ids = {}
ids = {}
for i in xrange(1024):
if trandom.random() > 0.1:
#add
id = None
while id is None or id in ids:
id = trandom.randint(0,1000000)
ids[id] = 1
t[id] = (id, ToBeDeleted(id), u'somename')
else:
#del
id = trandom.choice(ids.keys())
del t[id]
del ids[id]
ids = ids.keys()
trandom.shuffle(ids)
for id in ids:
del t[id]
ids = None
#to be on the safe side run a full GC
gc.collect()
#print LP294788_ids
self.assertEqual(len(t), 0)
self.assertEqual(len(LP294788_ids), 0)
# \\\
# /// BTree keys are objects, value is an int
t = OOBTree()
LP294788_ids = {}
ids = {}
for i in xrange(1024):
if trandom.random() > 0.1:
#add
id = None
while id is None or id in ids:
id = ToBeDeleted(trandom.randint(0,1000000))
ids[id] = 1
t[id] = 1
else:
#del
id = trandom.choice(ids.keys())
del ids[id]
del t[id]
ids = ids.keys()
trandom.shuffle(ids)
for id in ids:
del t[id]
#release all refs
ids = obj = id = None
#to be on the safe side run a full GC
gc.collect()
#print LP294788_ids
self.assertEqual(len(t), 0)
self.assertEqual(len(LP294788_ids), 0)
# /// BTree keys are tuples having objects, value is an int
t = OOBTree()
LP294788_ids = {}
ids = {}
for i in xrange(1024):
if trandom.random() > 0.1:
#add
id = None
while id is None or id in ids:
id = trandom.randint(0,1000000)
id = (id, ToBeDeleted(id), u'somename')
ids[id] = 1
t[id] = 1
else:
#del
id = trandom.choice(ids.keys())
del ids[id]
del t[id]
ids = ids.keys()
trandom.shuffle(ids)
for id in ids:
del t[id]
#release all refs
ids = id = obj = key = None
#to be on the safe side run a full GC
gc.collect()
#print LP294788_ids
self.assertEqual(len(t), 0)
self.assertEqual(len(LP294788_ids), 0)
class IIBTreeTest(BTreeTests):
def setUp(self):
self.t = IIBTree()
def testIIBTreeOverflow(self):
good = set()
b = self.t
def trial(i):
i = int(i)
try:
b[i] = 0
except TypeError:
self.assertRaises(TypeError, b.__setitem__, 0, i)
else:
good.add(i)
b[0] = i
self.assertEqual(b[0], i)
for i in range((1<<31) - 3, (1<<31) + 3):
trial(i)
trial(-i)
del b[0]
self.assertEqual(sorted(good), sorted(b))
class IFBTreeTest(BTreeTests):
def setUp(self):
self.t = IFBTree()
class IOBTreeTest(BTreeTests):
def setUp(self):
self.t = IOBTree()
class OIBTreeTest(BTreeTests):
def setUp(self):
self.t = OIBTree()
class OOBTreeTest(BTreeTests):
def setUp(self):
self.t = OOBTree()
def testRejectDefaultComparison(self):
# Check that passing int keys w default comparison fails.
# Only applies to new-style class instances. Old-style
# instances are too hard to introspect.
# This is white box because we know that the check is being
# used in a function that's used in lots of places.
# Otherwise, there are many permutations that would have to be
# checked.
class C(object):
pass
self.assertRaises(TypeError, lambda : self.t.__setitem__(C(), 1))
class C(object):
def __cmp__(*args):
return 1
c = C()
self.t[c] = 1
self.t.clear()
class C(object):
def __lt__(*args):
return 1
c = C()
self.t[c] = 1
self.t.clear()
if using64bits:
class IIBTreeTest(BTreeTests, TestLongIntKeys, TestLongIntValues):
def setUp(self):
self.t = IIBTree()
def getTwoValues(self):
return 1, 2
class IFBTreeTest(BTreeTests, TestLongIntKeys):
def setUp(self):
self.t = IFBTree()
def getTwoValues(self):
return 0.5, 1.5
class IOBTreeTest(BTreeTests, TestLongIntKeys):
def setUp(self):
self.t = IOBTree()
class OIBTreeTest(BTreeTests, TestLongIntValues):
def setUp(self):
self.t = OIBTree()
def getTwoKeys(self):
return object(), object()
class LLBTreeTest(BTreeTests, TestLongIntKeys, TestLongIntValues):
def setUp(self):
self.t = LLBTree()
def getTwoValues(self):
return 1, 2
class LFBTreeTest(BTreeTests, TestLongIntKeys):
def setUp(self):
self.t = LFBTree()
def getTwoValues(self):
return 0.5, 1.5
class LOBTreeTest(BTreeTests, TestLongIntKeys):
def setUp(self):
self.t = LOBTree()
class OLBTreeTest(BTreeTests, TestLongIntValues):
def setUp(self):
self.t = OLBTree()
def getTwoKeys(self):
return object(), object()
# cmp error propagation tests
class DoesntLikeBeingCompared:
def __cmp__(self,other):
raise ValueError('incomparable')
class TestCmpError(TestCase):
def testFoo(self):
t = OOBTree()
t['hello world'] = None
try:
t[DoesntLikeBeingCompared()] = None
except ValueError,e:
self.assertEqual(str(e), 'incomparable')
else:
self.fail('incomarable objects should not be allowed into '
'the tree')
# test for presence of generic names in module
class ModuleTest(TestCase):
module = None
prefix = None
iface = None
def testNames(self):
for name in ('Bucket', 'BTree', 'Set', 'TreeSet'):
klass = getattr(self.module, name)
self.assertEqual(klass.__module__, self.module.__name__)
self.assert_(klass is getattr(self.module, self.prefix + name))
def testModuleProvides(self):
self.assert_(
zope.interface.verify.verifyObject(self.iface, self.module))
def testFamily(self):
if self.prefix == 'OO':
self.assert_(
getattr(self.module, 'family', self) is self)
elif 'L' in self.prefix:
self.assert_(self.module.family is BTrees.family64)
elif 'I' in self.prefix:
self.assert_(self.module.family is BTrees.family32)
class FamilyTest(TestCase):
def test32(self):
self.assert_(
zope.interface.verify.verifyObject(
BTrees.Interfaces.IBTreeFamily, BTrees.family32))
self.assertEquals(
BTrees.family32.IO, BTrees.IOBTree)
self.assertEquals(
BTrees.family32.OI, BTrees.OIBTree)
self.assertEquals(
BTrees.family32.II, BTrees.IIBTree)
self.assertEquals(
BTrees.family32.IF, BTrees.IFBTree)
self.assertEquals(
BTrees.family32.OO, BTrees.OOBTree)
s = IOTreeSet()
s.insert(BTrees.family32.maxint)
self.assert_(BTrees.family32.maxint in s)
s = IOTreeSet()
s.insert(BTrees.family32.minint)
self.assert_(BTrees.family32.minint in s)
s = IOTreeSet()
# this next bit illustrates an, um, "interesting feature". If
# the characteristics change to match the 64 bit version, please
# feel free to change.
big = BTrees.family32.maxint + 1
self.assertRaises(TypeError, s.insert, big)
self.assertRaises(TypeError, s.insert, BTrees.family32.minint - 1)
self.check_pickling(BTrees.family32)
def test64(self):
self.assert_(
zope.interface.verify.verifyObject(
BTrees.Interfaces.IBTreeFamily, BTrees.family64))
self.assertEquals(
BTrees.family64.IO, BTrees.LOBTree)
self.assertEquals(
BTrees.family64.OI, BTrees.OLBTree)
self.assertEquals(
BTrees.family64.II, BTrees.LLBTree)
self.assertEquals(
BTrees.family64.IF, BTrees.LFBTree)
self.assertEquals(
BTrees.family64.OO, BTrees.OOBTree)
s = LOTreeSet()
s.insert(BTrees.family64.maxint)
self.assert_(BTrees.family64.maxint in s)
s = LOTreeSet()
s.insert(BTrees.family64.minint)
self.assert_(BTrees.family64.minint in s)
s = LOTreeSet()
self.assertRaises(ValueError, s.insert, BTrees.family64.maxint + 1)
self.assertRaises(ValueError, s.insert, BTrees.family64.minint - 1)
self.check_pickling(BTrees.family64)
def check_pickling(self, family):
# The "family" objects are singletons; they can be pickled and
# unpickled, and the same instances will always be returned on
# unpickling, whether from the same unpickler or different
# unpicklers.
s = pickle.dumps((family, family))
(f1, f2) = pickle.loads(s)
self.failUnless(f1 is family)
self.failUnless(f2 is family)
# Using a single memo across multiple pickles:
sio = StringIO.StringIO()
p = pickle.Pickler(sio)
p.dump(family)
p.dump([family])
u = pickle.Unpickler(StringIO.StringIO(sio.getvalue()))
f1 = u.load()
f2, = u.load()
self.failUnless(f1 is family)
self.failUnless(f2 is family)
# Using separate memos for each pickle:
sio = StringIO.StringIO()
p = pickle.Pickler(sio)
p.dump(family)
p.clear_memo()
p.dump([family])
u = pickle.Unpickler(StringIO.StringIO(sio.getvalue()))
f1 = u.load()
f2, = u.load()
self.failUnless(f1 is family)
self.failUnless(f2 is family)
class InternalKeysMappingTest(TestCase):
"""There must not be any internal keys not in the BTree
"""
def add_key(self, tree, key):
tree[key] = key
def test_internal_keys_after_deletion(self):
"""Make sure when a key's deleted, it's not an internal key
We'll leverage __getstate__ to introspect the internal structures.
We need to check BTrees with BTree children as well as BTrees
with bucket children.
"""
from ZODB.MappingStorage import DB
db = DB()
conn = db.open()
tree = conn.root.tree = self.t_class()
i = 0
# Grow the btree until we have multiple buckets
while 1:
i += 1
self.add_key(tree, i)
data = tree.__getstate__()[0]
if len(data) >= 3:
break
transaction.commit()
# Now, delete the internal key and make sure it's really gone
key = data[1]
del tree[key]
data = tree.__getstate__()[0]
self.assert_(data[1] != key)
# The tree should have changed:
self.assert_(tree._p_changed)
# Grow the btree until we have multiple levels
while 1:
i += 1
self.add_key(tree, i)
data = tree.__getstate__()[0]
if data[0].__class__ == tree.__class__:
assert len(data[2].__getstate__()[0]) >= 3
break
# Now, delete the internal key and make sure it's really gone
key = data[1]
del tree[key]
data = tree.__getstate__()[0]
self.assert_(data[1] != key)
transaction.abort()
db.close()
class InternalKeysSetTest:
"""There must not be any internal keys not in the TreeSet
"""
def add_key(self, tree, key):
tree.add(key)
def test_suite():
s = TestSuite()
for kv in ('OO',
'II', 'IO', 'OI', 'IF',
'LL', 'LO', 'OL', 'LF',
):
for name, bases in (
('BTree', (InternalKeysMappingTest,)),
('TreeSet', (InternalKeysSetTest,)),
):
klass = ClassType(kv + name + 'InternalKeyTest', bases,
dict(t_class=globals()[kv+name]))
s.addTest(makeSuite(klass))
for kv in ('OO',
'II', 'IO', 'OI', 'IF',
'LL', 'LO', 'OL', 'LF',
):
for name, bases in (
('Bucket', (MappingBase,)),
('TreeSet', (NormalSetTests,)),
('Set', (ExtendedSetTests,)),
):
klass = ClassType(kv + name + 'Test', bases,
dict(t_class=globals()[kv+name]))
s.addTest(makeSuite(klass))
for kv, iface in (
('OO', BTrees.Interfaces.IObjectObjectBTreeModule),
('IO', BTrees.Interfaces.IIntegerObjectBTreeModule),
('LO', BTrees.Interfaces.IIntegerObjectBTreeModule),
('OI', BTrees.Interfaces.IObjectIntegerBTreeModule),
('OL', BTrees.Interfaces.IObjectIntegerBTreeModule),
('II', BTrees.Interfaces.IIntegerIntegerBTreeModule),
('LL', BTrees.Interfaces.IIntegerIntegerBTreeModule),
('IF', BTrees.Interfaces.IIntegerFloatBTreeModule),
('LF', BTrees.Interfaces.IIntegerFloatBTreeModule)):
s.addTest(
makeSuite(
ClassType(
kv + 'ModuleTest',
(ModuleTest,),
dict(
prefix=kv,
module=getattr(BTrees, kv + 'BTree'),
iface=iface))))
for klass in (
IIBTreeTest, IFBTreeTest, IOBTreeTest, OIBTreeTest,
LLBTreeTest, LFBTreeTest, LOBTreeTest, OLBTreeTest,
OOBTreeTest,
# Note: there is no TestOOBTrees. The next three are
# checking for assorted TypeErrors, and when both keys
# and values are objects (OO), there's nothing to test.
TestIIBTrees, TestIFBTrees, TestIOBTrees, TestOIBTrees,
TestIOSets, TestIOTreeSets, TestIISets, TestIITreeSets,
TestLOSets, TestLOTreeSets, TestLLSets, TestLLTreeSets,
DegenerateBTree,
TestCmpError,
BugFixes,
FamilyTest,
):
s.addTest(makeSuite(klass))
return s
## utility functions
def lsubtract(l1, l2):
l1 = list(l1)
l2 = list(l2)
l = filter(lambda x, l1=l1: x not in l1, l2)
l = l + filter(lambda x, l2=l2: x not in l2, l1)
return l
def realseq(itemsob):
return [x for x in itemsob]
def permutations(x):
# Return a list of all permutations of list x.
n = len(x)
if n <= 1:
return [x]
result = []
x0 = x[0]
for i in range(n):
# Build the (n-1)! permutations with x[i] in the first position.
xcopy = x[:]
first, xcopy[i] = xcopy[i], x0
result.extend([[first] + p for p in permutations(xcopy[1:])])
return result
def main():
TextTestRunner().run(test_suite())
if __name__ == '__main__':
main()
BTrees/tests/testBTreesUnicode.py 0 → 100644
View file @ 703fc7e7
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import unittest
from BTrees.OOBTree import OOBTree
# When an OOBtree contains unicode strings as keys,
# it is neccessary accessing non-unicode strings are
# either ascii strings or encoded as unicoded using the
# corresponding encoding
encoding = 'ISO-8859-1'
class TestBTreesUnicode(unittest.TestCase):
""" test unicode"""
def setUp(self):
"""setup an OOBTree with some unicode strings"""
self.s = unicode('dreit\xe4gigen', 'latin1')
self.data = [('alien', 1),
('k\xf6nnten', 2),
('fox', 3),
('future', 4),
('quick', 5),
('zerst\xf6rt', 6),
(unicode('dreit\xe4gigen','latin1'), 7),
]
self.tree = OOBTree()
for k, v in self.data:
if isinstance(k, str):
k = unicode(k, 'latin1')
self.tree[k] = v
def testAllKeys(self):
# check every item of the tree
for k, v in self.data:
if isinstance(k, str):
k = unicode(k, encoding)
self.assert_(self.tree.has_key(k))
self.assertEqual(self.tree[k], v)
def testUnicodeKeys(self):
# try to access unicode keys in tree
k, v = self.data[-1]
self.assertEqual(k, self.s)
self.assertEqual(self.tree[k], v)
self.assertEqual(self.tree[self.s], v)
def testAsciiKeys(self):
# try to access some "plain ASCII" keys in the tree
for k, v in self.data[0], self.data[2]:
self.assert_(isinstance(k, str))
self.assertEqual(self.tree[k], v)
def test_suite():
return unittest.makeSuite(TestBTreesUnicode)
def main():
unittest.TextTestRunner().run(test_suite())
if __name__ == '__main__':
main()
BTrees/tests/testConflict.py 0 → 100644
View file @ 703fc7e7
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import os
from unittest import TestCase, TestSuite, makeSuite
from types import ClassType
from BTrees.OOBTree import OOBTree, OOBucket, OOSet, OOTreeSet
from BTrees.IOBTree import IOBTree, IOBucket, IOSet, IOTreeSet
from BTrees.IIBTree import IIBTree, IIBucket, IISet, IITreeSet
from BTrees.IFBTree import IFBTree, IFBucket, IFSet, IFTreeSet
from BTrees.OIBTree import OIBTree, OIBucket, OISet, OITreeSet
from BTrees.LOBTree import LOBTree, LOBucket, LOSet, LOTreeSet
from BTrees.LLBTree import LLBTree, LLBucket, LLSet, LLTreeSet
from BTrees.LFBTree import LFBTree, LFBucket, LFSet, LFTreeSet
from BTrees.OLBTree import OLBTree, OLBucket, OLSet, OLTreeSet
import transaction
from ZODB.POSException import ConflictError
class Base:
""" Tests common to all types: sets, buckets, and BTrees """
storage = None
def setUp(self):
self.t = self.t_type()
def tearDown(self):
transaction.abort()
del self.t
if self.storage is not None:
self.storage.close()
self.storage.cleanup()
def openDB(self):
from ZODB.FileStorage import FileStorage
from ZODB.DB import DB
n = 'fs_tmp__%s' % os.getpid()
self.storage = FileStorage(n)
self.db = DB(self.storage)
return self.db
class MappingBase(Base):
""" Tests common to mappings (buckets, btrees) """
def _deletefail(self):
del self.t[1]
def _setupConflict(self):
l=[ -5124, -7377, 2274, 8801, -9901, 7327, 1565, 17, -679,
3686, -3607, 14, 6419, -5637, 6040, -4556, -8622, 3847, 7191,
-4067]
e1=[(-1704, 0), (5420, 1), (-239, 2), (4024, 3), (-6984, 4)]
e2=[(7745, 0), (4868, 1), (-2548, 2), (-2711, 3), (-3154, 4)]
base=self.t
base.update([(i, i*i) for i in l[:20]])
b1=base.__class__(base)
b2=base.__class__(base)
bm=base.__class__(base)
items=base.items()
return base, b1, b2, bm, e1, e2, items
def testSimpleConflict(self):
# Unlike all the other tests, invoke conflict resolution
# by committing a transaction and catching a conflict
# in the storage.
self.openDB()
r1 = self.db.open().root()
r1["t"] = self.t
transaction.commit()
r2 = self.db.open().root()
copy = r2["t"]
list(copy) # unghostify
self.assertEqual(self.t._p_serial, copy._p_serial)
self.t.update({1:2, 2:3})
transaction.commit()
copy.update({3:4})
transaction.commit()
def testMergeDelete(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
del b1[items[1][0]]
del b2[items[5][0]]
del b1[items[-1][0]]
del b2[items[-2][0]]
del bm[items[1][0]]
del bm[items[5][0]]
del bm[items[-1][0]]
del bm[items[-2][0]]
test_merge(base, b1, b2, bm, 'merge delete')
def testMergeDeleteAndUpdate(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
del b1[items[1][0]]
b2[items[5][0]]=1
del b1[items[-1][0]]
b2[items[-2][0]]=2
del bm[items[1][0]]
bm[items[5][0]]=1
del bm[items[-1][0]]
bm[items[-2][0]]=2
test_merge(base, b1, b2, bm, 'merge update and delete')
def testMergeUpdate(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
b1[items[0][0]]=1
b2[items[5][0]]=2
b1[items[-1][0]]=3
b2[items[-2][0]]=4
bm[items[0][0]]=1
bm[items[5][0]]=2
bm[items[-1][0]]=3
bm[items[-2][0]]=4
test_merge(base, b1, b2, bm, 'merge update')
def testFailMergeDelete(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
del b1[items[0][0]]
del b2[items[0][0]]
test_merge(base, b1, b2, bm, 'merge conflicting delete',
should_fail=1)
def testFailMergeUpdate(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
b1[items[0][0]]=1
b2[items[0][0]]=2
test_merge(base, b1, b2, bm, 'merge conflicting update',
should_fail=1)
def testFailMergeDeleteAndUpdate(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
del b1[items[0][0]]
b2[items[0][0]]=-9
test_merge(base, b1, b2, bm, 'merge conflicting update and delete',
should_fail=1)
def testMergeInserts(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
b1[-99999]=-99999
b1[e1[0][0]]=e1[0][1]
b2[99999]=99999
b2[e1[2][0]]=e1[2][1]
bm[-99999]=-99999
bm[e1[0][0]]=e1[0][1]
bm[99999]=99999
bm[e1[2][0]]=e1[2][1]
test_merge(base, b1, b2, bm, 'merge insert')
def testMergeInsertsFromEmpty(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
base.clear()
b1.clear()
b2.clear()
bm.clear()
b1.update(e1)
bm.update(e1)
b2.update(e2)
bm.update(e2)
test_merge(base, b1, b2, bm, 'merge insert from empty')
def testFailMergeEmptyAndFill(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
b1.clear()
bm.clear()
b2.update(e2)
bm.update(e2)
test_merge(base, b1, b2, bm, 'merge insert from empty', should_fail=1)
def testMergeEmpty(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
b1.clear()
bm.clear()
test_merge(base, b1, b2, bm, 'empty one and not other', should_fail=1)
def testFailMergeInsert(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
b1[-99999]=-99999
b1[e1[0][0]]=e1[0][1]
b2[99999]=99999
b2[e1[0][0]]=e1[0][1]
test_merge(base, b1, b2, bm, 'merge conflicting inserts',
should_fail=1)
class SetTests(Base):
"Set (as opposed to TreeSet) specific tests."
def _setupConflict(self):
l=[ -5124, -7377, 2274, 8801, -9901, 7327, 1565, 17, -679,
3686, -3607, 14, 6419, -5637, 6040, -4556, -8622, 3847, 7191,
-4067]
e1=[-1704, 5420, -239, 4024, -6984]
e2=[7745, 4868, -2548, -2711, -3154]
base=self.t
base.update(l)
b1=base.__class__(base)
b2=base.__class__(base)
bm=base.__class__(base)
items=base.keys()
return base, b1, b2, bm, e1, e2, items
def testMergeDelete(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
b1.remove(items[1])
b2.remove(items[5])
b1.remove(items[-1])
b2.remove(items[-2])
bm.remove(items[1])
bm.remove(items[5])
bm.remove(items[-1])
bm.remove(items[-2])
test_merge(base, b1, b2, bm, 'merge delete')
def testFailMergeDelete(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
b1.remove(items[0])
b2.remove(items[0])
test_merge(base, b1, b2, bm, 'merge conflicting delete',
should_fail=1)
def testMergeInserts(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
b1.insert(-99999)
b1.insert(e1[0])
b2.insert(99999)
b2.insert(e1[2])
bm.insert(-99999)
bm.insert(e1[0])
bm.insert(99999)
bm.insert(e1[2])
test_merge(base, b1, b2, bm, 'merge insert')
def testMergeInsertsFromEmpty(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
base.clear()
b1.clear()
b2.clear()
bm.clear()
b1.update(e1)
bm.update(e1)
b2.update(e2)
bm.update(e2)
test_merge(base, b1, b2, bm, 'merge insert from empty')
def testFailMergeEmptyAndFill(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
b1.clear()
bm.clear()
b2.update(e2)
bm.update(e2)
test_merge(base, b1, b2, bm, 'merge insert from empty', should_fail=1)
def testMergeEmpty(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
b1.clear()
bm.clear()
test_merge(base, b1, b2, bm, 'empty one and not other', should_fail=1)
def testFailMergeInsert(self):
base, b1, b2, bm, e1, e2, items = self._setupConflict()
b1.insert(-99999)
b1.insert(e1[0])
b2.insert(99999)
b2.insert(e1[0])
test_merge(base, b1, b2, bm, 'merge conflicting inserts',
should_fail=1)
def test_merge(o1, o2, o3, expect, message='failed to merge', should_fail=0):
s1 = o1.__getstate__()
s2 = o2.__getstate__()
s3 = o3.__getstate__()
expected = expect.__getstate__()
if expected is None:
expected = ((((),),),)
if should_fail:
try:
merged = o1._p_resolveConflict(s1, s2, s3)
except ConflictError, err:
pass
else:
assert 0, message
else:
merged = o1._p_resolveConflict(s1, s2, s3)
assert merged == expected, message
class NastyConfict(Base, TestCase):
t_type = OOBTree
# This tests a problem that cropped up while trying to write
# testBucketSplitConflict (below): conflict resolution wasn't
# working at all in non-trivial cases. Symptoms varied from
# strange complaints about pickling (despite that the test isn't
# doing any *directly*), thru SystemErrors from Python and
# AssertionErrors inside the BTree code.
def testResolutionBlowsUp(self):
b = self.t
for i in range(0, 200, 4):
b[i] = i
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 64 .. 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# If these fail, the *preconditions* for running the test aren't
# satisfied -- the test itself hasn't been run yet.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
# Invoke conflict resolution by committing a transaction.
self.openDB()
r1 = self.db.open().root()
r1["t"] = self.t
transaction.commit()
r2 = self.db.open().root()
copy = r2["t"]
# Make sure all of copy is loaded.
list(copy.values())
self.assertEqual(self.t._p_serial, copy._p_serial)
self.t.update({1:2, 2:3})
transaction.commit()
copy.update({3:4})
transaction.commit() # if this doesn't blow up
list(copy.values()) # and this doesn't either, then fine
def testBucketSplitConflict(self):
# Tests that a bucket split is viewed as a conflict.
# It's (almost necessarily) a white-box test, and sensitive to
# implementation details.
b = self.t
for i in range(0, 200, 4):
b[i] = i
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 64 .. 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# If these fail, the *preconditions* for running the test aren't
# satisfied -- the test itself hasn't been run yet.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
# Invoke conflict resolution by committing a transaction.
self.openDB()
tm1 = transaction.TransactionManager()
r1 = self.db.open(transaction_manager=tm1).root()
r1["t"] = self.t
tm1.commit()
tm2 = transaction.TransactionManager()
r2 = self.db.open(transaction_manager=tm2).root()
copy = r2["t"]
# Make sure all of copy is loaded.
list(copy.values())
self.assertEqual(self.t._p_serial, copy._p_serial)
# In one transaction, add 16 new keys to bucket1, to force a bucket
# split.
b = self.t
numtoadd = 16
candidate = 60
while numtoadd:
if not b.has_key(candidate):
b[candidate] = candidate
numtoadd -= 1
candidate += 1
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 61 .. 74
# bucket 2 has 16 values: [75, 76 .. 81] + [84, 88 ..116]
# bucket 3 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((b0, 60, b1, 75, b2, 120, b3), firstbucket)
# The next block is still verifying preconditions.
self.assertEqual(len(state) , 2)
self.assertEqual(len(state[0]), 7)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 75)
self.assertEqual(state[0][5], 120)
tm1.commit()
# In the other transaction, add 3 values near the tail end of bucket1.
# This doesn't cause a split.
b = copy
for i in range(112, 116):
b[i] = i
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 18 values: 60, 64 .. 112, 113, 114, 115, 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# The next block is still verifying preconditions.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
self.assertRaises(ConflictError, tm2.commit)
def testEmptyBucketConflict(self):
# Tests that an emptied bucket *created by* conflict resolution is
# viewed as a conflict: conflict resolution doesn't have enough
# info to unlink the empty bucket from the BTree correctly.
b = self.t
for i in range(0, 200, 4):
b[i] = i
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 64 .. 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# If these fail, the *preconditions* for running the test aren't
# satisfied -- the test itself hasn't been run yet.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
# Invoke conflict resolution by committing a transaction.
self.openDB()
tm1 = transaction.TransactionManager()
r1 = self.db.open(transaction_manager=tm1).root()
r1["t"] = self.t
tm1.commit()
tm2 = transaction.TransactionManager()
r2 = self.db.open(transaction_manager=tm2).root()
copy = r2["t"]
# Make sure all of copy is loaded.
list(copy.values())
self.assertEqual(self.t._p_serial, copy._p_serial)
# In one transaction, delete half of bucket 1.
b = self.t
for k in 60, 64, 68, 72, 76, 80, 84, 88:
del b[k]
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 7 values: 92, 96, 100, 104, 108, 112, 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# The next block is still verifying preconditions.
self.assertEqual(len(state) , 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 92)
self.assertEqual(state[0][3], 120)
tm1.commit()
# In the other transaction, delete the other half of bucket 1.
b = copy
for k in 92, 96, 100, 104, 108, 112, 116:
del b[k]
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 8 values: 60, 64, 68, 72, 76, 80, 84, 88
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# The next block is still verifying preconditions.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
# Conflict resolution empties bucket1 entirely. This used to
# create an "insane" BTree (a legit BTree cannot contain an empty
# bucket -- it contains NULL pointers the BTree code doesn't
# expect, and segfaults result).
self.assertRaises(ConflictError, tm2.commit)
def testEmptyBucketNoConflict(self):
# Tests that a plain empty bucket (on input) is not viewed as a
# conflict.
b = self.t
for i in range(0, 200, 4):
b[i] = i
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 64 .. 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# If these fail, the *preconditions* for running the test aren't
# satisfied -- the test itself hasn't been run yet.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
# Invoke conflict resolution by committing a transaction.
self.openDB()
r1 = self.db.open().root()
r1["t"] = self.t
transaction.commit()
r2 = self.db.open().root()
copy = r2["t"]
# Make sure all of copy is loaded.
list(copy.values())
self.assertEqual(self.t._p_serial, copy._p_serial)
# In one transaction, just add a key.
b = self.t
b[1] = 1
# bucket 0 has 16 values: [0, 1] + [4, 8 .. 56]
# bucket 1 has 15 values: 60, 64 .. 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# The next block is still verifying preconditions.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
transaction.commit()
# In the other transaction, delete bucket 2.
b = copy
for k in range(120, 200, 4):
del b[k]
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 64 .. 116
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1), firstbucket)
# The next block is still verifying preconditions.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 3)
self.assertEqual(state[0][1], 60)
# This shouldn't create a ConflictError.
transaction.commit()
# And the resulting BTree shouldn't have internal damage.
b._check()
# The snaky control flow in _bucket__p_resolveConflict ended up trying
# to decref a NULL pointer if conflict resolution was fed 3 empty
# buckets. http://collector.zope.org/Zope/553
def testThreeEmptyBucketsNoSegfault(self):
self.t[1] = 1
bucket = self.t._firstbucket
del self.t[1]
state1 = bucket.__getstate__()
state2 = bucket.__getstate__()
state3 = bucket.__getstate__()
self.assert_(state2 is not state1 and
state2 is not state3 and
state3 is not state1)
self.assert_(state2 == state1 and
state3 == state1)
self.assertRaises(ConflictError, bucket._p_resolveConflict,
state1, state2, state3)
# When an empty BTree resolves conflicts, it computes the
# bucket state as None, so...
self.assertRaises(ConflictError, bucket._p_resolveConflict,
None, None, None)
def testCantResolveBTreeConflict(self):
# Test that a conflict involving two different changes to
# an internal BTree node is unresolvable. An internal node
# only changes when there are enough additions or deletions
# to a child bucket that the bucket is split or removed.
# It's (almost necessarily) a white-box test, and sensitive to
# implementation details.
b = self.t
for i in range(0, 200, 4):
b[i] = i
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 64 .. 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# If these fail, the *preconditions* for running the test aren't
# satisfied -- the test itself hasn't been run yet.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
# Set up database connections to provoke conflict.
self.openDB()
tm1 = transaction.TransactionManager()
r1 = self.db.open(transaction_manager=tm1).root()
r1["t"] = self.t
tm1.commit()
tm2 = transaction.TransactionManager()
r2 = self.db.open(transaction_manager=tm2).root()
copy = r2["t"]
# Make sure all of copy is loaded.
list(copy.values())
self.assertEqual(self.t._p_serial, copy._p_serial)
# Now one transaction should add enough keys to cause a split,
# and another should remove all the keys in one bucket.
for k in range(200, 300, 4):
self.t[k] = k
tm1.commit()
for k in range(0, 60, 4):
del copy[k]
try:
tm2.commit()
except ConflictError, detail:
self.assert_(str(detail).startswith('database conflict error'))
else:
self.fail("expected ConflictError")
def testConflictWithOneEmptyBucket(self):
# If one transaction empties a bucket, while another adds an item
# to the bucket, all the changes "look resolvable": bucket conflict
# resolution returns a bucket containing (only) the item added by
# the latter transaction, but changes from the former transaction
# removing the bucket are uncontested: the bucket is removed from
# the BTree despite that resolution thinks it's non-empty! This
# was first reported by Dieter Maurer, to zodb-dev on 22 Mar 2005.
b = self.t
for i in range(0, 200, 4):
b[i] = i
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 64 .. 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# If these fail, the *preconditions* for running the test aren't
# satisfied -- the test itself hasn't been run yet.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
# Set up database connections to provoke conflict.
self.openDB()
tm1 = transaction.TransactionManager()
r1 = self.db.open(transaction_manager=tm1).root()
r1["t"] = self.t
tm1.commit()
tm2 = transaction.TransactionManager()
r2 = self.db.open(transaction_manager=tm2).root()
copy = r2["t"]
# Make sure all of copy is loaded.
list(copy.values())
self.assertEqual(self.t._p_serial, copy._p_serial)
# Now one transaction empties the first bucket, and another adds a
# key to the first bucket.
for k in range(0, 60, 4):
del self.t[k]
tm1.commit()
copy[1] = 1
try:
tm2.commit()
except ConflictError, detail:
self.assert_(str(detail).startswith('database conflict error'))
else:
self.fail("expected ConflictError")
# Same thing, except commit the transactions in the opposite order.
b = OOBTree()
for i in range(0, 200, 4):
b[i] = i
tm1 = transaction.TransactionManager()
r1 = self.db.open(transaction_manager=tm1).root()
r1["t"] = b
tm1.commit()
tm2 = transaction.TransactionManager()
r2 = self.db.open(transaction_manager=tm2).root()
copy = r2["t"]
# Make sure all of copy is loaded.
list(copy.values())
self.assertEqual(b._p_serial, copy._p_serial)
# Now one transaction empties the first bucket, and another adds a
# key to the first bucket.
b[1] = 1
tm1.commit()
for k in range(0, 60, 4):
del copy[k]
try:
tm2.commit()
except ConflictError, detail:
self.assert_(str(detail).startswith('database conflict error'))
else:
self.fail("expected ConflictError")
def testConflictOfInsertAndDeleteOfFirstBucketItem(self):
"""
Recently, BTrees became careful about removing internal keys
(keys in internal aka BTree nodes) when they were deleted from
buckets. This poses a problem for conflict resolution.
We want to guard against a case in which the first key in a
bucket is removed in one transaction while a key is added
after that key but before the next key in another transaction
with the result that the added key is unreachble
original:
Bucket(...), k1, Bucket((k1, v1), (k3, v3), ...)
tran1
Bucket(...), k3, Bucket(k3, v3), ...)
tran2
Bucket(...), k1, Bucket((k1, v1), (k2, v2), (k3, v3), ...)
where k1 < k2 < k3
We don't want:
Bucket(...), k3, Bucket((k2, v2), (k3, v3), ...)
as k2 would be unfindable, so we want a conflict.
"""
mytype = self.t_type
db = self.openDB()
tm1 = transaction.TransactionManager()
conn1 = db.open(tm1)
conn1.root.t = t = mytype()
for i in range(0, 200, 2):
t[i] = i
tm1.commit()
k = t.__getstate__()[0][1]
assert t.__getstate__()[0][2].keys()[0] == k
tm2 = transaction.TransactionManager()
conn2 = db.open(tm2)
t[k+1] = k+1
del conn2.root.t[k]
for i in range(200,300):
conn2.root.t[i] = i
tm1.commit()
self.assertRaises(ConflictError, tm2.commit)
tm2.abort()
k = t.__getstate__()[0][1]
t[k+1] = k+1
del conn2.root.t[k]
tm2.commit()
self.assertRaises(ConflictError, tm1.commit)
tm1.abort()
def test_suite():
suite = TestSuite()
for kv in ('OO',
'II', 'IO', 'OI', 'IF',
'LL', 'LO', 'OL', 'LF',
):
for name, bases in (('BTree', (MappingBase, TestCase)),
('Bucket', (MappingBase, TestCase)),
('TreeSet', (SetTests, TestCase)),
('Set', (SetTests, TestCase)),
):
klass = ClassType(kv + name + 'Test', bases,
dict(t_type=globals()[kv+name]))
suite.addTest(makeSuite(klass))
suite.addTest(makeSuite(NastyConfict))
return suite
BTrees/tests/testLength.py 0 → 100644
View file @ 703fc7e7
##############################################################################
#
# Copyright (c) 2008 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""\
Test for BTrees.Length module.
"""
__docformat__ = "reStructuredText"
import BTrees.Length
import copy
import sys
import unittest
class LengthTestCase(unittest.TestCase):
def test_length_overflows_to_long(self):
length = BTrees.Length.Length(sys.maxint)
self.assertEqual(length(), sys.maxint)
self.assert_(type(length()) is int)
length.change(+1)
self.assertEqual(length(), sys.maxint + 1)
self.assert_(type(length()) is long)
def test_length_underflows_to_long(self):
minint = (-sys.maxint) - 1
length = BTrees.Length.Length(minint)
self.assertEqual(length(), minint)
self.assert_(type(length()) is int)
length.change(-1)
self.assertEqual(length(), minint - 1)
self.assert_(type(length()) is long)
def test_copy(self):
# Test for https://bugs.launchpad.net/zodb/+bug/516653
length = BTrees.Length.Length()
other = copy.copy(length)
self.assertEqual(other(), 0)
def test_suite():
return unittest.makeSuite(LengthTestCase)
BTrees/tests/testSetOps.py 0 → 100644
View file @ 703fc7e7
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
from unittest import TestCase, TestSuite, TextTestRunner, makeSuite
from BTrees.OOBTree import OOBTree, OOBucket, OOSet, OOTreeSet
from BTrees.IOBTree import IOBTree, IOBucket, IOSet, IOTreeSet
from BTrees.IFBTree import IFBTree, IFBucket, IFSet, IFTreeSet
from BTrees.IIBTree import IIBTree, IIBucket, IISet, IITreeSet
from BTrees.OIBTree import OIBTree, OIBucket, OISet, OITreeSet
from BTrees.LOBTree import LOBTree, LOBucket, LOSet, LOTreeSet
from BTrees.LFBTree import LFBTree, LFBucket, LFSet, LFTreeSet
from BTrees.LLBTree import LLBTree, LLBucket, LLSet, LLTreeSet
from BTrees.OLBTree import OLBTree, OLBucket, OLSet, OLTreeSet
# Subclasses have to set up:
# builders - functions to build inputs, taking an optional keys arg
# intersection, union, difference - set to the type-correct versions
class SetResult(TestCase):
def setUp(self):
self.Akeys = [1, 3, 5, 6 ]
self.Bkeys = [ 2, 3, 4, 6, 7]
self.As = [makeset(self.Akeys) for makeset in self.builders]
self.Bs = [makeset(self.Bkeys) for makeset in self.builders]
self.emptys = [makeset() for makeset in self.builders]
# Slow but obviously correct Python implementations of basic ops.
def _union(self, x, y):
result = list(x)
for e in y:
if e not in result:
result.append(e)
result.sort()
return result
def _intersection(self, x, y):
result = []
for e in x:
if e in y:
result.append(e)
return result
def _difference(self, x, y):
result = list(x)
for e in y:
if e in result:
result.remove(e)
# Difference preserves LHS values.
if hasattr(x, "values"):
result = [(k, x[k]) for k in result]
return result
def testNone(self):
for op in self.union, self.intersection, self.difference:
C = op(None, None)
self.assert_(C is None)
for op in self.union, self.intersection, self.difference:
for A in self.As:
C = op(A, None)
self.assert_(C is A)
C = op(None, A)
if op is self.difference:
self.assert_(C is None)
else:
self.assert_(C is A)
def testEmptyUnion(self):
for A in self.As:
for E in self.emptys:
C = self.union(A, E)
self.assert_(not hasattr(C, "values"))
self.assertEqual(list(C), self.Akeys)
C = self.union(E, A)
self.assert_(not hasattr(C, "values"))
self.assertEqual(list(C), self.Akeys)
def testEmptyIntersection(self):
for A in self.As:
for E in self.emptys:
C = self.intersection(A, E)
self.assert_(not hasattr(C, "values"))
self.assertEqual(list(C), [])
C = self.intersection(E, A)
self.assert_(not hasattr(C, "values"))
self.assertEqual(list(C), [])
def testEmptyDifference(self):
for A in self.As:
for E in self.emptys:
C = self.difference(A, E)
# Difference preserves LHS values.
self.assertEqual(hasattr(C, "values"), hasattr(A, "values"))
if hasattr(A, "values"):
self.assertEqual(list(C.items()), list(A.items()))
else:
self.assertEqual(list(C), self.Akeys)
C = self.difference(E, A)
self.assertEqual(hasattr(C, "values"), hasattr(E, "values"))
self.assertEqual(list(C), [])
def testUnion(self):
inputs = self.As + self.Bs
for A in inputs:
for B in inputs:
C = self.union(A, B)
self.assert_(not hasattr(C, "values"))
self.assertEqual(list(C), self._union(A, B))
def testIntersection(self):
inputs = self.As + self.Bs
for A in inputs:
for B in inputs:
C = self.intersection(A, B)
self.assert_(not hasattr(C, "values"))
self.assertEqual(list(C), self._intersection(A, B))
def testDifference(self):
inputs = self.As + self.Bs
for A in inputs:
for B in inputs:
C = self.difference(A, B)
# Difference preserves LHS values.
self.assertEqual(hasattr(C, "values"), hasattr(A, "values"))
want = self._difference(A, B)
if hasattr(A, "values"):
self.assertEqual(list(C.items()), want)
else:
self.assertEqual(list(C), want)
def testLargerInputs(self):
from random import randint
MAXSIZE = 200
MAXVAL = 400
for i in range(3):
n = randint(0, MAXSIZE)
Akeys = [randint(1, MAXVAL) for j in range(n)]
As = [makeset(Akeys) for makeset in self.builders]
Akeys = IISet(Akeys)
n = randint(0, MAXSIZE)
Bkeys = [randint(1, MAXVAL) for j in range(n)]
Bs = [makeset(Bkeys) for makeset in self.builders]
Bkeys = IISet(Bkeys)
for op, simulator in ((self.union, self._union),
(self.intersection, self._intersection),
(self.difference, self._difference)):
for A in As:
for B in Bs:
got = op(A, B)
want = simulator(Akeys, Bkeys)
self.assertEqual(list(got), want,
(A, B, Akeys, Bkeys, list(got), want))
# Given a mapping builder (IIBTree, OOBucket, etc), return a function
# that builds an object of that type given only a list of keys.
def makeBuilder(mapbuilder):
def result(keys=[], mapbuilder=mapbuilder):
return mapbuilder(zip(keys, keys))
return result
class PureOO(SetResult):
from BTrees.OOBTree import union, intersection, difference
builders = OOSet, OOTreeSet, makeBuilder(OOBTree), makeBuilder(OOBucket)
class PureII(SetResult):
from BTrees.IIBTree import union, intersection, difference
builders = IISet, IITreeSet, makeBuilder(IIBTree), makeBuilder(IIBucket)
class PureIO(SetResult):
from BTrees.IOBTree import union, intersection, difference
builders = IOSet, IOTreeSet, makeBuilder(IOBTree), makeBuilder(IOBucket)
class PureIF(SetResult):
from BTrees.IFBTree import union, intersection, difference
builders = IFSet, IFTreeSet, makeBuilder(IFBTree), makeBuilder(IFBucket)
class PureOI(SetResult):
from BTrees.OIBTree import union, intersection, difference
builders = OISet, OITreeSet, makeBuilder(OIBTree), makeBuilder(OIBucket)
class PureLL(SetResult):
from BTrees.LLBTree import union, intersection, difference
builders = LLSet, LLTreeSet, makeBuilder(LLBTree), makeBuilder(LLBucket)
class PureLO(SetResult):
from BTrees.LOBTree import union, intersection, difference
builders = LOSet, LOTreeSet, makeBuilder(LOBTree), makeBuilder(LOBucket)
class PureLF(SetResult):
from BTrees.LFBTree import union, intersection, difference
builders = LFSet, LFTreeSet, makeBuilder(LFBTree), makeBuilder(LFBucket)
class PureOL(SetResult):
from BTrees.OLBTree import union, intersection, difference
builders = OLSet, OLTreeSet, makeBuilder(OLBTree), makeBuilder(OLBucket)
# Subclasses must set up (as class variables):
# multiunion, union
# mkset, mktreeset
# mkbucket, mkbtree
class MultiUnion(TestCase):
def testEmpty(self):
self.assertEqual(len(self.multiunion([])), 0)
def testOne(self):
for sequence in [3], range(20), range(-10, 0, 2) + range(1, 10, 2):
seq1 = sequence[:]
seq2 = sequence[:]
seq2.reverse()
seqsorted = sequence[:]
seqsorted.sort()
for seq in seq1, seq2, seqsorted:
for builder in self.mkset, self.mktreeset:
input = builder(seq)
output = self.multiunion([input])
self.assertEqual(len(seq), len(output))
self.assertEqual(seqsorted, list(output))
def testValuesIgnored(self):
for builder in self.mkbucket, self.mkbtree:
input = builder([(1, 2), (3, 4), (5, 6)])
output = self.multiunion([input])
self.assertEqual([1, 3, 5], list(output))
def testBigInput(self):
N = 100000
input = self.mkset(range(N))
output = self.multiunion([input] * 10)
self.assertEqual(len(output), N)
self.assertEqual(output.minKey(), 0)
self.assertEqual(output.maxKey(), N-1)
self.assertEqual(list(output), range(N))
def testLotsOfLittleOnes(self):
from random import shuffle
N = 5000
inputs = []
mkset, mktreeset = self.mkset, self.mktreeset
for i in range(N):
base = i * 4 - N
inputs.append(mkset([base, base+1]))
inputs.append(mktreeset([base+2, base+3]))
shuffle(inputs)
output = self.multiunion(inputs)
self.assertEqual(len(output), N*4)
self.assertEqual(list(output), range(-N, 3*N))
def testFunkyKeyIteration(self):
# The internal set iteration protocol allows "iterating over" a
# a single key as if it were a set.
N = 100
union, mkset = self.union, self.mkset
slow = mkset()
for i in range(N):
slow = union(slow, mkset([i]))
fast = self.multiunion(range(N)) # acts like N distinct singleton sets
self.assertEqual(len(slow), N)
self.assertEqual(len(fast), N)
self.assertEqual(list(slow), list(fast))
self.assertEqual(list(fast), range(N))
class TestIIMultiUnion(MultiUnion):
from BTrees.IIBTree import multiunion, union
from BTrees.IIBTree import IISet as mkset, IITreeSet as mktreeset
from BTrees.IIBTree import IIBucket as mkbucket, IIBTree as mkbtree
class TestIOMultiUnion(MultiUnion):
from BTrees.IOBTree import multiunion, union
from BTrees.IOBTree import IOSet as mkset, IOTreeSet as mktreeset
from BTrees.IOBTree import IOBucket as mkbucket, IOBTree as mkbtree
class TestIFMultiUnion(MultiUnion):
from BTrees.IFBTree import multiunion, union
from BTrees.IFBTree import IFSet as mkset, IFTreeSet as mktreeset
from BTrees.IFBTree import IFBucket as mkbucket, IFBTree as mkbtree
class TestLLMultiUnion(MultiUnion):
from BTrees.LLBTree import multiunion, union
from BTrees.LLBTree import LLSet as mkset, LLTreeSet as mktreeset
from BTrees.LLBTree import LLBucket as mkbucket, LLBTree as mkbtree
class TestLOMultiUnion(MultiUnion):
from BTrees.LOBTree import multiunion, union
from BTrees.LOBTree import LOSet as mkset, LOTreeSet as mktreeset
from BTrees.LOBTree import LOBucket as mkbucket, LOBTree as mkbtree
class TestLFMultiUnion(MultiUnion):
from BTrees.LFBTree import multiunion, union
from BTrees.LFBTree import LFSet as mkset, LFTreeSet as mktreeset
from BTrees.LFBTree import LFBucket as mkbucket, LFBTree as mkbtree
# Check that various special module functions are and aren't imported from
# the expected BTree modules.
class TestImports(TestCase):
def testWeightedUnion(self):
from BTrees.IIBTree import weightedUnion
from BTrees.OIBTree import weightedUnion
try:
from BTrees.IOBTree import weightedUnion
except ImportError:
pass
else:
self.fail("IOBTree shouldn't have weightedUnion")
from BTrees.LLBTree import weightedUnion
from BTrees.OLBTree import weightedUnion
try:
from BTrees.LOBTree import weightedUnion
except ImportError:
pass
else:
self.fail("LOBTree shouldn't have weightedUnion")
try:
from BTrees.OOBTree import weightedUnion
except ImportError:
pass
else:
self.fail("OOBTree shouldn't have weightedUnion")
def testWeightedIntersection(self):
from BTrees.IIBTree import weightedIntersection
from BTrees.OIBTree import weightedIntersection
try:
from BTrees.IOBTree import weightedIntersection
except ImportError:
pass
else:
self.fail("IOBTree shouldn't have weightedIntersection")
from BTrees.LLBTree import weightedIntersection
from BTrees.OLBTree import weightedIntersection
try:
from BTrees.LOBTree import weightedIntersection
except ImportError:
pass
else:
self.fail("LOBTree shouldn't have weightedIntersection")
try:
from BTrees.OOBTree import weightedIntersection
except ImportError:
pass
else:
self.fail("OOBTree shouldn't have weightedIntersection")
def testMultiunion(self):
from BTrees.IIBTree import multiunion
from BTrees.IOBTree import multiunion
try:
from BTrees.OIBTree import multiunion
except ImportError:
pass
else:
self.fail("OIBTree shouldn't have multiunion")
from BTrees.LLBTree import multiunion
from BTrees.LOBTree import multiunion
try:
from BTrees.OLBTree import multiunion
except ImportError:
pass
else:
self.fail("OLBTree shouldn't have multiunion")
try:
from BTrees.OOBTree import multiunion
except ImportError:
pass
else:
self.fail("OOBTree shouldn't have multiunion")
# Subclasses must set up (as class variables):
# weightedUnion, weightedIntersection
# builders -- sequence of constructors, taking items
# union, intersection -- the module routines of those names
# mkbucket -- the module bucket builder
class Weighted(TestCase):
def setUp(self):
self.Aitems = [(1, 10), (3, 30), (5, 50), (6, 60)]
self.Bitems = [(2, 21), (3, 31), (4, 41), (6, 61), (7, 71)]
self.As = [make(self.Aitems) for make in self.builders]
self.Bs = [make(self.Bitems) for make in self.builders]
self.emptys = [make([]) for make in self.builders]
weights = []
for w1 in -3, -1, 0, 1, 7:
for w2 in -3, -1, 0, 1, 7:
weights.append((w1, w2))
self.weights = weights
def testBothNone(self):
for op in self.weightedUnion, self.weightedIntersection:
w, C = op(None, None)
self.assert_(C is None)
self.assertEqual(w, 0)
w, C = op(None, None, 42, 666)
self.assert_(C is None)
self.assertEqual(w, 0)
def testLeftNone(self):
for op in self.weightedUnion, self.weightedIntersection:
for A in self.As + self.emptys:
w, C = op(None, A)
self.assert_(C is A)
self.assertEqual(w, 1)
w, C = op(None, A, 42, 666)
self.assert_(C is A)
self.assertEqual(w, 666)
def testRightNone(self):
for op in self.weightedUnion, self.weightedIntersection:
for A in self.As + self.emptys:
w, C = op(A, None)
self.assert_(C is A)
self.assertEqual(w, 1)
w, C = op(A, None, 42, 666)
self.assert_(C is A)
self.assertEqual(w, 42)
# If obj is a set, return a bucket with values all 1; else return obj.
def _normalize(self, obj):
if isaset(obj):
obj = self.mkbucket(zip(obj, [1] * len(obj)))
return obj
# Python simulation of weightedUnion.
def _wunion(self, A, B, w1=1, w2=1):
if isaset(A) and isaset(B):
return 1, self.union(A, B).keys()
A = self._normalize(A)
B = self._normalize(B)
result = []
for key in self.union(A, B):
v1 = A.get(key, 0)
v2 = B.get(key, 0)
result.append((key, v1*w1 + v2*w2))
return 1, result
def testUnion(self):
inputs = self.As + self.Bs + self.emptys
for A in inputs:
for B in inputs:
want_w, want_s = self._wunion(A, B)
got_w, got_s = self.weightedUnion(A, B)
self.assertEqual(got_w, want_w)
if isaset(got_s):
self.assertEqual(got_s.keys(), want_s)
else:
self.assertEqual(got_s.items(), want_s)
for w1, w2 in self.weights:
want_w, want_s = self._wunion(A, B, w1, w2)
got_w, got_s = self.weightedUnion(A, B, w1, w2)
self.assertEqual(got_w, want_w)
if isaset(got_s):
self.assertEqual(got_s.keys(), want_s)
else:
self.assertEqual(got_s.items(), want_s)
# Python simulation weightedIntersection.
def _wintersection(self, A, B, w1=1, w2=1):
if isaset(A) and isaset(B):
return w1 + w2, self.intersection(A, B).keys()
A = self._normalize(A)
B = self._normalize(B)
result = []
for key in self.intersection(A, B):
result.append((key, A[key]*w1 + B[key]*w2))
return 1, result
def testIntersection(self):
inputs = self.As + self.Bs + self.emptys
for A in inputs:
for B in inputs:
want_w, want_s = self._wintersection(A, B)
got_w, got_s = self.weightedIntersection(A, B)
self.assertEqual(got_w, want_w)
if isaset(got_s):
self.assertEqual(got_s.keys(), want_s)
else:
self.assertEqual(got_s.items(), want_s)
for w1, w2 in self.weights:
want_w, want_s = self._wintersection(A, B, w1, w2)
got_w, got_s = self.weightedIntersection(A, B, w1, w2)
self.assertEqual(got_w, want_w)
if isaset(got_s):
self.assertEqual(got_s.keys(), want_s)
else:
self.assertEqual(got_s.items(), want_s)
# Given a set builder (like OITreeSet or OISet), return a function that
# takes a list of (key, value) pairs and builds a set out of the keys.
def itemsToSet(setbuilder):
def result(items, setbuilder=setbuilder):
return setbuilder([key for key, value in items])
return result
class TestWeightedII(Weighted):
from BTrees.IIBTree import weightedUnion, weightedIntersection
from BTrees.IIBTree import union, intersection
from BTrees.IIBTree import IIBucket as mkbucket
builders = IIBucket, IIBTree, itemsToSet(IISet), itemsToSet(IITreeSet)
class TestWeightedOI(Weighted):
from BTrees.OIBTree import weightedUnion, weightedIntersection
from BTrees.OIBTree import union, intersection
from BTrees.OIBTree import OIBucket as mkbucket
builders = OIBucket, OIBTree, itemsToSet(OISet), itemsToSet(OITreeSet)
class TestWeightedLL(Weighted):
from BTrees.LLBTree import weightedUnion, weightedIntersection
from BTrees.LLBTree import union, intersection
from BTrees.LLBTree import LLBucket as mkbucket
builders = LLBucket, LLBTree, itemsToSet(LLSet), itemsToSet(LLTreeSet)
class TestWeightedOL(Weighted):
from BTrees.OLBTree import weightedUnion, weightedIntersection
from BTrees.OLBTree import union, intersection
from BTrees.OLBTree import OLBucket as mkbucket
builders = OLBucket, OLBTree, itemsToSet(OLSet), itemsToSet(OLTreeSet)
# 'thing' is a bucket, btree, set or treeset. Return true iff it's one of the
# latter two.
def isaset(thing):
return not hasattr(thing, 'values')
def test_suite():
s = TestSuite()
for klass in (
TestIIMultiUnion, TestIOMultiUnion, TestIFMultiUnion,
TestLLMultiUnion, TestLOMultiUnion, TestLFMultiUnion,
TestImports,
PureOO,
PureII, PureIO, PureIF, PureOI,
PureLL, PureLO, PureLF, PureOL,
TestWeightedII, TestWeightedOI,
TestWeightedLL, TestWeightedOL,
):
s.addTest(makeSuite(klass))
return s
def main():
TextTestRunner().run(test_suite())
if __name__ == '__main__':
main()
BTrees/tests/test_btreesubclass.py 0 → 100644
View file @ 703fc7e7
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
from BTrees.OOBTree import OOBTree, OOBucket
class B(OOBucket):
pass
class T(OOBTree):
_bucket_type = B
import unittest
class SubclassTest(unittest.TestCase):
def testSubclass(self):
# test that a subclass that defines _bucket_type gets buckets
# of that type
t = T()
# There's no good way to get a bucket at the moment.
# __getstate__() is as good as it gets, but the default
# getstate explicitly includes the pickle of the bucket
# for small trees, so we have to be clever :-(
# make sure there is more than one bucket in the tree
for i in range(1000):
t[i] = i
state = t.__getstate__()
self.assert_(state[0][0].__class__ is B)
def test_suite():
return unittest.makeSuite(SubclassTest)
BTrees/tests/test_check.py 0 → 100644
View file @ 703fc7e7
##############################################################################
#
# Copyright (c) 2003 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Test the BTree check.check() function."""
import unittest
from BTrees.OOBTree import OOBTree
from BTrees.check import check
class CheckTest(unittest.TestCase):
def setUp(self):
self.t = t = OOBTree()
for i in range(31):
t[i] = 2*i
self.state = t.__getstate__()
def testNormal(self):
s = self.state
# Looks like (state, first_bucket)
# where state looks like (bucket0, 15, bucket1).
self.assertEqual(len(s), 2)
self.assertEqual(len(s[0]), 3)
self.assertEqual(s[0][1], 15)
self.t._check() # shouldn't blow up
check(self.t) # shouldn't blow up
def testKeyTooLarge(self):
# Damage an invariant by dropping the BTree key to 14.
s = self.state
news = (s[0][0], 14, s[0][2]), s[1]
self.t.__setstate__(news)
self.t._check() # not caught
try:
# Expecting "... key %r >= upper bound %r at index %d"
check(self.t)
except AssertionError, detail:
self.failUnless(str(detail).find(">= upper bound") > 0)
else:
self.fail("expected self.t_check() to catch the problem")
def testKeyTooSmall(self):
# Damage an invariant by bumping the BTree key to 16.
s = self.state
news = (s[0][0], 16, s[0][2]), s[1]
self.t.__setstate__(news)
self.t._check() # not caught
try:
# Expecting "... key %r < lower bound %r at index %d"
check(self.t)
except AssertionError, detail:
self.failUnless(str(detail).find("< lower bound") > 0)
else:
self.fail("expected self.t_check() to catch the problem")
def testKeysSwapped(self):
# Damage an invariant by swapping two key/value pairs.
s = self.state
# Looks like (state, first_bucket)
# where state looks like (bucket0, 15, bucket1).
(b0, num, b1), firstbucket = s
self.assertEqual(b0[4], 8)
self.assertEqual(b0[5], 10)
b0state = b0.__getstate__()
self.assertEqual(len(b0state), 2)
# b0state looks like
# ((k0, v0, k1, v1, ...), nextbucket)
pairs, nextbucket = b0state
self.assertEqual(pairs[8], 4)
self.assertEqual(pairs[9], 8)
self.assertEqual(pairs[10], 5)
self.assertEqual(pairs[11], 10)
newpairs = pairs[:8] + (5, 10, 4, 8) + pairs[12:]
b0.__setstate__((newpairs, nextbucket))
self.t._check() # not caught
try:
check(self.t)
except AssertionError, detail:
self.failUnless(str(detail).find(
"key 5 at index 4 >= key 4 at index 5") > 0)
else:
self.fail("expected self.t_check() to catch the problem")
def test_suite():
return unittest.makeSuite(CheckTest)
BTrees/tests/test_compare.py 0 → 100644
View file @ 703fc7e7
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Test errors during comparison of BTree keys."""
import unittest
from BTrees.OOBTree import OOBucket as Bucket, OOSet as Set
import transaction
from ZODB.MappingStorage import MappingStorage
from ZODB.DB import DB
class CompareTest(unittest.TestCase):
s = "A string with hi-bit-set characters: \700\701"
u = u"A unicode string"
def setUp(self):
# These defaults only make sense if the default encoding
# prevents s from being promoted to Unicode.
self.assertRaises(UnicodeError, unicode, self.s)
# An object needs to be added to the database to
self.db = DB(MappingStorage())
root = self.db.open().root()
self.bucket = root["bucket"] = Bucket()
self.set = root["set"] = Set()
transaction.commit()
def tearDown(self):
self.assert_(self.bucket._p_changed != 2)
self.assert_(self.set._p_changed != 2)
transaction.abort()
def assertUE(self, callable, *args):
self.assertRaises(UnicodeError, callable, *args)
def testBucketGet(self):
import sys
import warnings
_warnlog = []
def _showwarning(*args, **kw):
_warnlog.append((args, kw))
warnings.showwarning, _before = _showwarning, warnings.showwarning
try:
self.bucket[self.s] = 1
self.assertUE(self.bucket.get, self.u)
finally:
warnings.showwarning = _before
if sys.version_info >= (2, 6):
self.assertEqual(len(_warnlog), 1)
def testSetGet(self):
self.set.insert(self.s)
self.assertUE(self.set.remove, self.u)
def testBucketSet(self):
self.bucket[self.s] = 1
self.assertUE(self.bucket.__setitem__, self.u, 1)
def testSetSet(self):
self.set.insert(self.s)
self.assertUE(self.set.insert, self.u)
def testBucketMinKey(self):
self.bucket[self.s] = 1
self.assertUE(self.bucket.minKey, self.u)
def testSetMinKey(self):
self.set.insert(self.s)
self.assertUE(self.set.minKey, self.u)
def test_suite():
return unittest.makeSuite(CompareTest)
BTrees/tests/test_fsBTree.py 0 → 100644
View file @ 703fc7e7
##############################################################################
#
# Copyright (c) 2010 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
import doctest
import unittest
def test_fsbucket_string_conversion():
"""
fsBuckets have toString and fromString methods that can be used to
get and set their state very efficiently:
>>> from BTrees.fsBTree import fsBucket
>>> b = fsBucket([(c*2, c*6) for c in 'abcdef'])
>>> import pprint
>>> b.toString()
'aabbccddeeffaaaaaabbbbbbccccccddddddeeeeeeffffff'
>>> b2 = fsBucket().fromString(b.toString())
>>> b.__getstate__() == b2.__getstate__()
True
"""
def test_suite():
return doctest.DocTestSuite()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7