tdb: delete from CCAN.

This was an early experiment in putting code into CCAN, but TDB is a public
library distributed as part of the Samba sources; there is no reason to
publish it here (especially now the unit tests are also in Samba).

ccan/tdb/LICENSE

-../../licenses/LGPL-3
\ No newline at end of file

ccan/tdb/_info

-#include <string.h>
-#include <stdio.h>
-
-/**
- * tdb - The trivial (transactional) database
- *
- * The tdb module provides an efficient keyword data mapping (usually
- * within a file).  It supports transactions, so the contents of the
- * database is reliable even across crashes.
- *
- * Example:
- *	#include <ccan/tdb/tdb.h>
- *	#include <ccan/str/str.h>
- *	#include <err.h>
- *	#include <stdio.h>
- *	
- *	static void usage(const char *argv0)
- *	{
- *		errx(1, "Usage: %s fetch <dbfile> <key>\n"
- *		     "OR %s store <dbfile> <key> <data>", argv0, argv0);
- *	}
- *	
- *	int main(int argc, char *argv[])
- *	{
- *		struct tdb_context *tdb;
- *		TDB_DATA key, value;
- *	
- *		if (argc < 4)
- *			usage(argv[0]);
- *	
- *		tdb = tdb_open(argv[2], 1024, TDB_DEFAULT, O_CREAT|O_RDWR,
- *				0600);
- *		if (!tdb)
- *			err(1, "Opening %s", argv[2]);
- *	
- *		key.dptr = (void *)argv[3];
- *		key.dsize = strlen(argv[3]);
- *	
- *		if (streq(argv[1], "fetch")) {
- *			if (argc != 4)
- *				usage(argv[0]);
- *			value = tdb_fetch(tdb, key);
- *			if (!value.dptr)
- *				errx(1, "fetch %s: %s",
- *				     argv[3], tdb_errorstr(tdb));
- *			printf("%.*s\n", value.dsize, (char *)value.dptr);
- *			free(value.dptr);
- *		} else if (streq(argv[1], "store")) {
- *			if (argc != 5)
- *				usage(argv[0]);
- *			value.dptr = (void *)argv[4];
- *			value.dsize = strlen(argv[4]);
- *			if (tdb_store(tdb, key, value, 0) != 0)
- *				errx(1, "store %s: %s",
- *				     argv[3], tdb_errorstr(tdb));
- *		} else
- *			usage(argv[0]);
- *	
- *		return 0;
- *	}
- *
- * Maintainer: Rusty Russell <rusty@rustcorp.com.au>
- *
- * Author: Andrew Tridgell, Jeremy Allison, Rusty Russell
- *
- * License: LGPL (v3 or any later version)
- *
- * Ccanlint:
- *	// valgrind breaks fcntl locks.
- *	tests_pass_valgrind test/run-open-during-transaction.c:FAIL
- *	tests_pass_valgrind_noleaks test/run-die-during-transaction.c:FAIL
- */
-int main(int argc, char *argv[])
-{
-	if (argc != 2)
-		return 1;
-
-	if (strcmp(argv[1], "depends") == 0) {
-		printf("ccan/compiler\n");
-		return 0;
-	}
-
-	return 1;
-}

ccan/tdb/dump.c

- /* 
-   Unix SMB/CIFS implementation.
-
-   trivial database library
-
-   Copyright (C) Andrew Tridgell              1999-2005
-   Copyright (C) Paul `Rusty' Russell		   2000
-   Copyright (C) Jeremy Allison			   2000-2003
-   
-     ** NOTE! The following LGPL license applies to the tdb
-     ** library. This does NOT imply that all of Samba is released
-     ** under the LGPL
-   
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include "tdb_private.h"
-
-static tdb_off_t tdb_dump_record(struct tdb_context *tdb, int hash,
-				 tdb_off_t offset)
-{
-	struct tdb_record rec;
-	tdb_off_t tailer_ofs, tailer;
-
-	if (tdb->methods->tdb_read(tdb, offset, (char *)&rec, 
-				   sizeof(rec), DOCONV()) == -1) {
-		printf("ERROR: failed to read record at %u\n", offset);
-		return 0;
-	}
-
-	printf(" rec: hash=%d offset=0x%08x next=0x%08x rec_len=%d "
-	       "key_len=%d data_len=%d full_hash=0x%x magic=0x%x\n",
-	       hash, offset, rec.next, rec.rec_len, rec.key_len, rec.data_len,
-	       rec.full_hash, rec.magic);
-
-	tailer_ofs = offset + sizeof(rec) + rec.rec_len - sizeof(tdb_off_t);
-
-	if (tdb_ofs_read(tdb, tailer_ofs, &tailer) == -1) {
-		printf("ERROR: failed to read tailer at %u\n", tailer_ofs);
-		return rec.next;
-	}
-
-	if (tailer != rec.rec_len + sizeof(rec)) {
-		printf("ERROR: tailer does not match record! tailer=%u totalsize=%u\n",
-				(unsigned int)tailer, (unsigned int)(rec.rec_len + sizeof(rec)));
-	}
-	return rec.next;
-}
-
-static int tdb_dump_chain(struct tdb_context *tdb, int i)
-{
-	tdb_off_t rec_ptr, top;
-
-	top = TDB_HASH_TOP(i);
-
-	if (tdb_lock(tdb, i, F_WRLCK) != 0)
-		return -1;
-
-	if (tdb_ofs_read(tdb, top, &rec_ptr) == -1)
-		return tdb_unlock(tdb, i, F_WRLCK);
-
-	if (rec_ptr)
-		printf("hash=%d\n", i);
-
-	while (rec_ptr) {
-		rec_ptr = tdb_dump_record(tdb, i, rec_ptr);
-	}
-
-	return tdb_unlock(tdb, i, F_WRLCK);
-}
-
-void tdb_dump_all(struct tdb_context *tdb)
-{
-	int i;
-	for (i=0;i<tdb->header.hash_size;i++) {
-		tdb_dump_chain(tdb, i);
-	}
-	printf("freelist:\n");
-	tdb_dump_chain(tdb, -1);
-}
-
-int tdb_printfreelist(struct tdb_context *tdb)
-{
-	int ret;
-	long total_free = 0;
-	tdb_off_t offset, rec_ptr;
-	struct tdb_record rec;
-
-	if ((ret = tdb_lock(tdb, -1, F_WRLCK)) != 0)
-		return ret;
-
-	offset = FREELIST_TOP;
-
-	/* read in the freelist top */
-	if (tdb_ofs_read(tdb, offset, &rec_ptr) == -1) {
-		tdb_unlock(tdb, -1, F_WRLCK);
-		return 0;
-	}
-
-	printf("freelist top=[0x%08x]\n", rec_ptr );
-	while (rec_ptr) {
-		if (tdb->methods->tdb_read(tdb, rec_ptr, (char *)&rec, 
-					   sizeof(rec), DOCONV()) == -1) {
-			tdb_unlock(tdb, -1, F_WRLCK);
-			return -1;
-		}
-
-		if (rec.magic != TDB_FREE_MAGIC) {
-			printf("bad magic 0x%08x in free list\n", rec.magic);
-			tdb_unlock(tdb, -1, F_WRLCK);
-			return -1;
-		}
-
-		printf("entry offset=[0x%08x], rec.rec_len = [0x%08x (%d)] (end = 0x%08x)\n", 
-		       rec_ptr, rec.rec_len, rec.rec_len, rec_ptr + rec.rec_len);
-		total_free += rec.rec_len;
-
-		/* move to the next record */
-		rec_ptr = rec.next;
-	}
-	printf("total rec_len = [0x%08x (%d)]\n", (int)total_free, 
-               (int)total_free);
-
-	return tdb_unlock(tdb, -1, F_WRLCK);
-}
-

ccan/tdb/error.c

- /* 
-   Unix SMB/CIFS implementation.
-
-   trivial database library
-
-   Copyright (C) Andrew Tridgell              1999-2005
-   Copyright (C) Paul `Rusty' Russell		   2000
-   Copyright (C) Jeremy Allison			   2000-2003
-   
-     ** NOTE! The following LGPL license applies to the tdb
-     ** library. This does NOT imply that all of Samba is released
-     ** under the LGPL
-   
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include "tdb_private.h"
-
-enum TDB_ERROR tdb_error(struct tdb_context *tdb)
-{
-	return tdb->ecode;
-}
-
-static struct tdb_errname {
-	enum TDB_ERROR ecode; const char *estring;
-} emap[] = { {TDB_SUCCESS, "Success"},
-	     {TDB_ERR_CORRUPT, "Corrupt database"},
-	     {TDB_ERR_IO, "IO Error"},
-	     {TDB_ERR_LOCK, "Locking error"},
-	     {TDB_ERR_OOM, "Out of memory"},
-	     {TDB_ERR_EXISTS, "Record exists"},
-	     {TDB_ERR_NOLOCK, "Lock exists on other keys"},
-	     {TDB_ERR_EINVAL, "Invalid parameter"},
-	     {TDB_ERR_NOEXIST, "Record does not exist"},
-	     {TDB_ERR_RDONLY, "write not permitted"} };
-
-/* Error string for the last tdb error */
-const char *tdb_errorstr(struct tdb_context *tdb)
-{
-	uint32_t i;
-	for (i = 0; i < sizeof(emap) / sizeof(struct tdb_errname); i++)
-		if (tdb->ecode == emap[i].ecode)
-			return emap[i].estring;
-	return "Invalid error code";
-}
-

ccan/tdb/freelistcheck.c

-/*
-   Unix SMB/CIFS implementation.
-
-   trivial database library
-
-   Copyright (C) Jeremy Allison                    2006
-
-     ** NOTE! The following LGPL license applies to the tdb
-     ** library. This does NOT imply that all of Samba is released
-     ** under the LGPL
-
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include "tdb_private.h"
-
-/* Check the freelist is good and contains no loops.
-   Very memory intensive - only do this as a consistency
-   checker. Heh heh - uses an in memory tdb as the storage
-   for the "seen" record list. For some reason this strikes
-   me as extremely clever as I don't have to write another tree
-   data structure implementation :-).
- */
-
-static int seen_insert(struct tdb_context *mem_tdb, tdb_off_t rec_ptr)
-{
-	TDB_DATA key, data;
-
-	memset(&data, '\0', sizeof(data));
-	key.dptr = (unsigned char *)&rec_ptr;
-	key.dsize = sizeof(rec_ptr);
-	return tdb_store(mem_tdb, key, data, TDB_INSERT);
-}
-
-int tdb_validate_freelist(struct tdb_context *tdb, int *pnum_entries)
-{
-	struct tdb_context *mem_tdb = NULL;
-	struct tdb_record rec;
-	tdb_off_t rec_ptr;
-	int ret = -1;
-
-	*pnum_entries = 0;
-
-	mem_tdb = tdb_open("flval", tdb->header.hash_size,
-				TDB_INTERNAL, O_RDWR, 0600);
-	if (!mem_tdb) {
-		return -1;
-	}
-
-	if (tdb_lock(tdb, -1, F_WRLCK) == -1) {
-		tdb_close(mem_tdb);
-		return 0;
-	}
-
-	/* Store the FREELIST_TOP record. */
-	if (seen_insert(mem_tdb, FREELIST_TOP) == -1) {
-		tdb->ecode = TDB_ERR_CORRUPT;
-		ret = -1;
-		goto fail;
-	}
-
-	/* read in the freelist top */
-	if (tdb_ofs_read(tdb, FREELIST_TOP, &rec_ptr) == -1) {
-		goto fail;
-	}
-
-	while (rec_ptr) {
-
-		/* If we can't store this record (we've seen it
-		   before) then the free list has a loop and must
-		   be corrupt. */
-
-		if (seen_insert(mem_tdb, rec_ptr)) {
-			tdb->ecode = TDB_ERR_CORRUPT;
-			ret = -1;
-			goto fail;
-		}
-
-		if (tdb_rec_free_read(tdb, rec_ptr, &rec) == -1) {
-			goto fail;
-		}
-
-		/* move to the next record */
-		rec_ptr = rec.next;
-		*pnum_entries += 1;
-	}
-
-	ret = 0;
-
-  fail:
-
-	tdb_close(mem_tdb);
-	tdb_unlock(tdb, -1, F_WRLCK);
-	return ret;
-}

ccan/tdb/summary.c

- /* 
-   Trivial Database: human-readable summary code
-   Copyright (C) Rusty Russell 2010
-   
-   This library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 3 of the License, or (at your option) any later version.
-
-   This library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, see <http://www.gnu.org/licenses/>.
-*/
-#include "tdb_private.h"
-
-#define SUMMARY_FORMAT \
-	"Size of file/data: %u/%zu\n" \
-	"Number of records: %zu\n" \
-	"Smallest/average/largest keys: %zu/%zu/%zu\n" \
-	"Smallest/average/largest data: %zu/%zu/%zu\n" \
-	"Smallest/average/largest padding: %zu/%zu/%zu\n" \
-	"Number of dead records: %zu\n" \
-	"Smallest/average/largest dead records: %zu/%zu/%zu\n" \
-	"Number of free records: %zu\n" \
-	"Smallest/average/largest free records: %zu/%zu/%zu\n" \
-	"Number of hash chains: %zu\n" \
-	"Smallest/average/largest hash chains: %zu/%zu/%zu\n" \
-	"Number of uncoalesced records: %zu\n" \
-	"Smallest/average/largest uncoalesced runs: %zu/%zu/%zu\n" \
-	"Percentage keys/data/padding/free/dead/rechdrs&tailers/hashes: %.0f/%.0f/%.0f/%.0f/%.0f/%.0f/%.0f\n"
-
-/* We don't use tally module, to keep upstream happy. */
-struct tally {
-	size_t min, max, total;
-	size_t num;
-};
-
-static void tally_init(struct tally *tally)
-{
-	tally->total = 0;
-	tally->num = 0;
-	tally->min = tally->max = 0;
-}
-
-static void tally_add(struct tally *tally, size_t len)
-{
-	if (tally->num == 0)
-		tally->max = tally->min = len;
-	else if (len > tally->max)
-		tally->max = len;
-	else if (len < tally->min)
-		tally->min = len;
-	tally->num++;
-	tally->total += len;
-}
-
-static size_t tally_mean(const struct tally *tally)
-{
-	if (!tally->num)
-		return 0;
-	return tally->total / tally->num;
-}
-
-static size_t get_hash_length(struct tdb_context *tdb, unsigned int i)
-{
-	tdb_off_t rec_ptr;
-	size_t count = 0;
-
-	if (tdb_ofs_read(tdb, TDB_HASH_TOP(i), &rec_ptr) == -1)
-		return 0;
-
-	/* keep looking until we find the right record */
-	while (rec_ptr) {
-		struct tdb_record r;
-		++count;
-		if (tdb_rec_read(tdb, rec_ptr, &r) == -1)
-			return 0;
-		rec_ptr = r.next;
-	}
-	return count;
-}
-
-char *tdb_summary(struct tdb_context *tdb)
-{
-	tdb_off_t off;
-	struct tally freet, keys, data, dead, extra, hash, uncoal;
-	struct tdb_record rec;
-	char *ret = NULL;
-	bool locked;
-	size_t len, unc = 0;
-
-	/* Read-only databases use no locking at all: it's best-effort.
-	 * We may have a write lock already, so skip that case too. */
-	if (tdb->read_only || tdb->allrecord_lock.count != 0) {
-		locked = false;
-	} else {
-		if (tdb_lockall_read(tdb) == -1)
-			return NULL;
-		locked = true;
-	}
-
-	tally_init(&freet);
-	tally_init(&keys);
-	tally_init(&data);
-	tally_init(&dead);
-	tally_init(&extra);
-	tally_init(&hash);
-	tally_init(&uncoal);
-
-	for (off = TDB_DATA_START(tdb->header.hash_size);
-	     off < tdb->map_size - 1;
-	     off += sizeof(rec) + rec.rec_len) {
-		if (tdb->methods->tdb_read(tdb, off, &rec, sizeof(rec),
-					   DOCONV()) == -1)
-			goto unlock;
-		switch (rec.magic) {
-		case TDB_MAGIC:
-			tally_add(&keys, rec.key_len);
-			tally_add(&data, rec.data_len);
-			tally_add(&extra, rec.rec_len - (rec.key_len
-							 + rec.data_len));
-			if (unc > 1)
-				tally_add(&uncoal, unc - 1);
-			unc = 0;
-			break;
-		case TDB_FREE_MAGIC:
-			tally_add(&freet, rec.rec_len);
-			unc++;
-			break;
-		/* If we crash after ftruncate, we can get zeroes or fill. */
-		case TDB_RECOVERY_INVALID_MAGIC:
-		case 0x42424242:
-			unc++;
-			rec.rec_len = tdb_dead_space(tdb, off) - sizeof(rec);
-			/* Fall through */
-		case TDB_DEAD_MAGIC:
-			tally_add(&dead, rec.rec_len);
-			break;
-		default:
-			TDB_LOG((tdb, TDB_DEBUG_ERROR,
-				 "Unexpected record magic 0x%x at offset %d\n",
-				 rec.magic, off));
-			goto unlock;
-		}
-	}
-	if (unc > 1)
-		tally_add(&uncoal, unc - 1);
-
-	for (off = 0; off < tdb->header.hash_size; off++)
-		tally_add(&hash, get_hash_length(tdb, off));
-
-	/* 20 is max length of a %zu. */
-	len = strlen(SUMMARY_FORMAT) + 35*20 + 1;
-	ret = malloc(len);
-	if (!ret)
-		goto unlock;
-
-	sprintf(ret, SUMMARY_FORMAT,
-		tdb->map_size, keys.total+data.total,
-		keys.num,
-		keys.min, tally_mean(&keys), keys.max,
-		data.min, tally_mean(&data), data.max,
-		extra.min, tally_mean(&extra), extra.max,
-		dead.num,
-		dead.min, tally_mean(&dead), dead.max,
-		freet.num,
-		freet.min, tally_mean(&freet), freet.max,
-		hash.num,
-		hash.min, tally_mean(&hash), hash.max,
-		uncoal.total,
-		uncoal.min, tally_mean(&uncoal), uncoal.max,
-		keys.total * 100.0 / tdb->map_size,
-		data.total * 100.0 / tdb->map_size,
-		extra.total * 100.0 / tdb->map_size,
-		freet.total * 100.0 / tdb->map_size,
-		dead.total * 100.0 / tdb->map_size,
-		(keys.num + freet.num + dead.num)
-		* (sizeof(struct tdb_record) + sizeof(uint32_t))
-		* 100.0 / tdb->map_size,
-		tdb->header.hash_size * sizeof(tdb_off_t)
-		* 100.0 / tdb->map_size);
-
-unlock:
-	if (locked) {
-		tdb_unlockall_read(tdb);
-	}
-	return ret;
-}

ccan/tdb/test/external-agent.h

-#ifndef TDB_TEST_EXTERNAL_AGENT_H
-#define TDB_TEST_EXTERNAL_AGENT_H
-
-/* For locking tests, we need a different process to try things at
- * various times. */
-enum operation {
-	OPEN,
-	OPEN_WITH_CLEAR_IF_FIRST,
-	TRANSACTION_START,
-	FETCH,
-	STORE,
-	TRANSACTION_COMMIT,
-	CHECK,
-	NEEDS_RECOVERY,
-	CLOSE,
-};
-
-/* Do this before doing any tdb stuff.  Return handle, or -1. */
-struct agent *prepare_external_agent(void);
-
-enum agent_return {
-	SUCCESS,
-	WOULD_HAVE_BLOCKED,
-	AGENT_DIED,
-	FAILED, /* For fetch, or NEEDS_RECOVERY */
-	OTHER_FAILURE,
-};
-
-/* Ask the external agent to try to do an operation. 
- * name == tdb name for OPEN/OPEN_WITH_CLEAR_IF_FIRST,
- * record name for FETCH/STORE (store stores name as data too)
- */
-enum agent_return external_agent_operation(struct agent *handle,
-					   enum operation op,
-					   const char *name);
-
-/* Mapping enum -> string. */
-const char *agent_return_name(enum agent_return ret);
-const char *operation_name(enum operation op);
-
-#endif /* TDB_TEST_EXTERNAL_AGENT_H */

ccan/tdb/test/logging.h

-#ifndef TDB_TEST_LOGGING_H
-#define TDB_TEST_LOGGING_H
-#include <ccan/tdb/tdb.h>
-#include <stdbool.h>
-
-extern bool suppress_logging;
-extern const char *log_prefix;
-extern struct tdb_logging_context taplogctx;
-
-#endif /* TDB_TEST_LOGGING_H */