From 95a72ac82a4ddd0fc8f27cc487e292ee70fa9c97 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Fri, 18 Nov 2011 15:43:55 -0800 Subject: [PATCH 1/8] Add $(SOLIBS) macro for shared lib dependencies --- libraries/libmdb/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/libmdb/Makefile b/libraries/libmdb/Makefile index e49e275638..88297078e1 100644 --- a/libraries/libmdb/Makefile +++ b/libraries/libmdb/Makefile @@ -3,6 +3,7 @@ W = -W -Wall -Wno-unused-parameter -Wbad-function-cast OPT = -O2 -g CFLAGS = -pthread $(OPT) $(W) $(XCFLAGS) LDLIBS = +SOLIBS = PROGS = mdb_stat mtest mtest2 mtest3 mtest4 mtest5 all: libmdb.a libmdb.so $(PROGS) @@ -18,7 +19,7 @@ libmdb.a: mdb.o midl.o ar rs $@ mdb.o midl.o libmdb.so: mdb.o midl.o - gcc -shared -o $@ mdb.o midl.o + gcc -shared -o $@ mdb.o midl.o $(SOLIBS) mdb_stat: mdb_stat.o libmdb.a mtest: mtest.o libmdb.a From 81afbb2fca0effc3637c88ce03935b6b3f5eca47 Mon Sep 17 00:00:00 2001 From: Kurt Zeilenga Date: Sun, 1 Jan 2012 07:07:45 -0800 Subject: [PATCH 2/8] Happy New Year! --- libraries/libmdb/midl.c | 2 +- libraries/libmdb/midl.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/libmdb/midl.c b/libraries/libmdb/midl.c index 500811f7d2..a2cb861659 100644 --- a/libraries/libmdb/midl.c +++ b/libraries/libmdb/midl.c @@ -3,7 +3,7 @@ /* $OpenLDAP$ */ /* This work is part of OpenLDAP Software . * - * Copyright 2000-2011 The OpenLDAP Foundation. + * Copyright 2000-2012 The OpenLDAP Foundation. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/libraries/libmdb/midl.h b/libraries/libmdb/midl.h index 4c0e615f1a..87c8768031 100644 --- a/libraries/libmdb/midl.h +++ b/libraries/libmdb/midl.h @@ -11,7 +11,7 @@ /* $OpenLDAP$ */ /* This work is part of OpenLDAP Software . * - * Copyright 2000-2011 The OpenLDAP Foundation. + * Copyright 2000-2012 The OpenLDAP Foundation. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From a17f8e73265d5407e0a3f98ab79add0f839b997b Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Wed, 4 Jan 2012 15:25:18 -0800 Subject: [PATCH 3/8] ITS#7121 fix page splits when data+key size is too large --- libraries/libmdb/mdb.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libraries/libmdb/mdb.c b/libraries/libmdb/mdb.c index bec30d3e06..d953ef941a 100644 --- a/libraries/libmdb/mdb.c +++ b/libraries/libmdb/mdb.c @@ -4342,7 +4342,7 @@ mdb_leaf_size(MDB_env *env, MDB_val *key, MDB_val *data) size_t sz; sz = LEAFSIZE(key, data); - if (data->mv_size >= env->me_psize / MDB_MINKEYS) { + if (sz >= env->me_psize / MDB_MINKEYS) { /* put on overflow page */ sz -= data->mv_size - sizeof(pgno_t); } @@ -4435,11 +4435,11 @@ mdb_node_add(MDB_cursor *mc, indx_t indx, if (F_ISSET(flags, F_BIGDATA)) { /* Data already on overflow page. */ node_size += sizeof(pgno_t); - } else if (data->mv_size >= mc->mc_txn->mt_env->me_psize / MDB_MINKEYS) { + } else if (node_size + data->mv_size >= mc->mc_txn->mt_env->me_psize / MDB_MINKEYS) { int ovpages = OVPAGES(data->mv_size, mc->mc_txn->mt_env->me_psize); /* Put data on overflow page. */ - DPRINTF("data size is %zu, put on overflow page", - data->mv_size); + DPRINTF("data size is %zu, node would be %zu, put data on overflow page", + data->mv_size, node_size+data->mv_size); node_size += sizeof(pgno_t); if ((ofp = mdb_page_new(mc, P_OVERFLOW, ovpages)) == NULL) return ENOMEM; From 713bf092b6d3ac350deea6d8f641110784d61efe Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Sat, 21 Jan 2012 08:07:56 -0800 Subject: [PATCH 4/8] Happy New Year! --- libraries/libmdb/mdb.c | 2 +- libraries/libmdb/mdb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/libmdb/mdb.c b/libraries/libmdb/mdb.c index d953ef941a..adf98befde 100644 --- a/libraries/libmdb/mdb.c +++ b/libraries/libmdb/mdb.c @@ -5,7 +5,7 @@ * BerkeleyDB API, but much simplified. */ /* - * Copyright 2011 Howard Chu, Symas Corp. + * Copyright 2011-2012 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/libraries/libmdb/mdb.h b/libraries/libmdb/mdb.h index 9075c41e0c..c40a5e9e28 100644 --- a/libraries/libmdb/mdb.h +++ b/libraries/libmdb/mdb.h @@ -33,7 +33,7 @@ * * @author Howard Chu, Symas Corporation. * - * @copyright Copyright 2011 Howard Chu, Symas Corp. All rights reserved. + * @copyright Copyright 2011-2012 Howard Chu, Symas Corp. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted only as authorized by the OpenLDAP From b9cbcb05889811f64983622cd75a3485f688a0dc Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Sat, 21 Jan 2012 08:12:56 -0800 Subject: [PATCH 5/8] ITS#7126 fix cursor tracking code Don't compare cursors that aren't deep enough to match the current cursor. --- libraries/libmdb/mdb.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/libraries/libmdb/mdb.c b/libraries/libmdb/mdb.c index adf98befde..cdd46c98d7 100644 --- a/libraries/libmdb/mdb.c +++ b/libraries/libmdb/mdb.c @@ -1234,6 +1234,7 @@ finish: for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { if (m2 == mc) continue; m3 = &m2->mc_xcursor->mx_cursor; + if (m3->mc_snum < mc->mc_snum) continue; if (m3->mc_pg[mc->mc_top] == mc->mc_pg[mc->mc_top]) { m3->mc_pg[mc->mc_top] = mp; } @@ -1242,7 +1243,7 @@ finish: MDB_cursor *m2; for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { - if (m2 == mc) continue; + if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; if (m2->mc_pg[mc->mc_top] == mc->mc_pg[mc->mc_top]) { m2->mc_pg[mc->mc_top] = mp; } @@ -4167,7 +4168,7 @@ new_sub: m3 = &m2->mc_xcursor->mx_cursor; else m3 = m2; - if (m3 == mc) continue; + if (m3 == mc || m3->mc_snum < mc->mc_snum) continue; if (m3->mc_pg[i] == mp && m3->mc_ki[i] >= mc->mc_ki[i]) { m3->mc_ki[i]++; } @@ -4208,7 +4209,7 @@ put_sub: MDB_page *mp = mc->mc_pg[i]; for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { - if (m2 == mc) continue; + if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; if (m2->mc_pg[i] == mp && m2->mc_ki[i] == mc->mc_ki[i]) { mdb_xcursor_init1(m2, leaf); } @@ -4713,6 +4714,7 @@ mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx) mc->mc_dbx = &txn->mt_dbxs[dbi]; mc->mc_dbflag = &txn->mt_dbflags[dbi]; mc->mc_snum = 0; + mc->mc_top = 0; mc->mc_flags = 0; if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) { assert(mx != NULL); @@ -5076,6 +5078,7 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst) m3 = &m2->mc_xcursor->mx_cursor; else m3 = m2; + if (m3->mc_snum < csrc->mc_snum) continue; if (m3->mc_pg[csrc->mc_top] == csrc->mc_pg[csrc->mc_top]) { m3->mc_pg[csrc->mc_top] = mp; m3->mc_ki[csrc->mc_top] += nkeys; @@ -5167,6 +5170,7 @@ mdb_rebalance(MDB_cursor *mc) m3 = &m2->mc_xcursor->mx_cursor; else m3 = m2; + if (m3->mc_snum < mc->mc_snum) continue; if (m3->mc_pg[0] == mp) { m3->mc_snum = 0; m3->mc_top = 0; @@ -5196,6 +5200,7 @@ mdb_rebalance(MDB_cursor *mc) m3 = &m2->mc_xcursor->mx_cursor; else m3 = m2; + if (m3->mc_snum < mc->mc_snum) continue; if (m3->mc_pg[0] == mp) { m3->mc_pg[0] = mc->mc_pg[0]; } From 1c8de1883c1a9c239a5ba69a291c07889c1fe7da Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Sat, 21 Jan 2012 08:18:37 -0800 Subject: [PATCH 6/8] ITS#7121 fix page splits when newindx == 0 --- libraries/libmdb/mdb.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libraries/libmdb/mdb.c b/libraries/libmdb/mdb.c index cdd46c98d7..9480a4339e 100644 --- a/libraries/libmdb/mdb.c +++ b/libraries/libmdb/mdb.c @@ -5585,6 +5585,8 @@ newsep: ins_new = 1; /* Update page and index for the new key. */ + if (!newindx) + mc->mc_pg[mc->mc_top] = copy; mc->mc_ki[mc->mc_top] = j; } else if (i == nkeys) { break; @@ -5620,7 +5622,7 @@ newsep: mc->mc_txn->mt_env->me_psize - copy->mp_upper); /* reset back to original page */ - if (newindx < split_indx) { + if (!newindx || (newindx < split_indx)) { mc->mc_pg[mc->mc_top] = mp; if (nflags & MDB_RESERVE) { node = NODEPTR(mp, mc->mc_ki[mc->mc_top]); From 3d6b924c2560f39973de82d2599ce9d6542b9cb2 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Sat, 21 Jan 2012 08:20:43 -0800 Subject: [PATCH 7/8] txn_commit must abort txn before returning errors Fix for failure when committing a txn that is a child of some parent txn. --- libraries/libmdb/mdb.c | 4 +++- libraries/libmdb/mdb.h | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/libraries/libmdb/mdb.c b/libraries/libmdb/mdb.c index 9480a4339e..cd0e810bae 100644 --- a/libraries/libmdb/mdb.c +++ b/libraries/libmdb/mdb.c @@ -1737,8 +1737,10 @@ mdb_txn_commit(MDB_txn *txn) } x = dst[0].mid; for (; y<=src[0].mid; y++) { - if (++x >= MDB_IDL_UM_MAX) + if (++x >= MDB_IDL_UM_MAX) { + mdb_txn_abort(txn); return ENOMEM; + } dst[x] = src[y]; } dst[0].mid = x; diff --git a/libraries/libmdb/mdb.h b/libraries/libmdb/mdb.h index c40a5e9e28..5682c8f868 100644 --- a/libraries/libmdb/mdb.h +++ b/libraries/libmdb/mdb.h @@ -530,6 +530,7 @@ int mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn ** *
  • EINVAL - an invalid parameter was specified. *
  • ENOSPC - no more disk space. *
  • EIO - a low-level I/O error occurred while writing. + *
  • ENOMEM - the transaction is nested and could not be merged into its parent. * */ int mdb_txn_commit(MDB_txn *txn); From e8e57be42107a877f604d0c4e67115ef37681a1b Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Sat, 21 Jan 2012 08:22:59 -0800 Subject: [PATCH 8/8] Add valgrind support with -DUSE_VALGRIND Track allocations and invalid accesses to freed tmp pages. --- libraries/libmdb/mdb.c | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/libraries/libmdb/mdb.c b/libraries/libmdb/mdb.c index cd0e810bae..03cf70769e 100644 --- a/libraries/libmdb/mdb.c +++ b/libraries/libmdb/mdb.c @@ -64,6 +64,21 @@ #endif #endif +#ifdef USE_VALGRIND +#include +#define VGMEMP_CREATE(h,r,z) VALGRIND_CREATE_MEMPOOL(h,r,z) +#define VGMEMP_ALLOC(h,a,s) VALGRIND_MEMPOOL_ALLOC(h,a,s) +#define VGMEMP_FREE(h,a) VALGRIND_MEMPOOL_FREE(h,a) +#define VGMEMP_DESTROY(h) VALGRIND_DESTROY_MEMPOOL(h) +#define VGMEMP_DEFINED(a,s) VALGRIND_MAKE_MEM_DEFINED(a,s) +#else +#define VGMEMP_CREATE(h,r,z) +#define VGMEMP_ALLOC(h,a,s) +#define VGMEMP_FREE(h,a) +#define VGMEMP_DESTROY(h) +#define VGMEMP_DEFINED(a,s) +#endif + #ifndef BYTE_ORDER # if (defined(_LITTLE_ENDIAN) || defined(_BIG_ENDIAN)) && !(defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN)) /* Solaris just defines one or the other */ @@ -1068,11 +1083,15 @@ mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b) static MDB_page * mdb_page_malloc(MDB_cursor *mc) { MDB_page *ret; + size_t sz = mc->mc_txn->mt_env->me_psize; if (mc->mc_txn->mt_env->me_dpages) { ret = mc->mc_txn->mt_env->me_dpages; + VGMEMP_ALLOC(mc->mc_txn->mt_env, ret, sz); + VGMEMP_DEFINED(ret, sizeof(ret->mp_next)); mc->mc_txn->mt_env->me_dpages = ret->mp_next; } else { - ret = malloc(mc->mc_txn->mt_env->me_psize); + ret = malloc(sz); + VGMEMP_ALLOC(mc->mc_txn->mt_env, ret, sz); } return ret; } @@ -1184,10 +1203,14 @@ mdb_page_alloc(MDB_cursor *mc, int num) } if (txn->mt_env->me_dpages && num == 1) { np = txn->mt_env->me_dpages; + VGMEMP_ALLOC(txn->mt_env, np, txn->mt_env->me_psize); + VGMEMP_DEFINED(np, sizeof(np->mp_next)); txn->mt_env->me_dpages = np->mp_next; } else { - if ((np = malloc(txn->mt_env->me_psize * num )) == NULL) + size_t sz = txn->mt_env->me_psize * num; + if ((np = malloc(sz)) == NULL) return NULL; + VGMEMP_ALLOC(txn->mt_env, np, sz); } if (pgno == P_INVALID) { np->mp_pgno = txn->mt_next_pgno; @@ -1590,9 +1613,11 @@ mdb_txn_reset0(MDB_txn *txn) dp = txn->mt_u.dirty_list[i].mptr; if (!IS_OVERFLOW(dp) || dp->mp_pages == 1) { dp->mp_next = txn->mt_env->me_dpages; + VGMEMP_FREE(txn->mt_env, dp); txn->mt_env->me_dpages = dp; } else { /* large pages just get freed directly */ + VGMEMP_FREE(txn->mt_env, dp); free(dp); } } @@ -1943,8 +1968,10 @@ mdb_txn_commit(MDB_txn *txn) dp = txn->mt_u.dirty_list[i].mptr; if (!IS_OVERFLOW(dp) || dp->mp_pages == 1) { dp->mp_next = txn->mt_env->me_dpages; + VGMEMP_FREE(txn->mt_env, dp); txn->mt_env->me_dpages = dp; } else { + VGMEMP_FREE(txn->mt_env, dp); free(dp); } txn->mt_u.dirty_list[i].mid = 0; @@ -2221,6 +2248,7 @@ mdb_env_create(MDB_env **env) e->me_fd = INVALID_HANDLE_VALUE; e->me_lfd = INVALID_HANDLE_VALUE; e->me_mfd = INVALID_HANDLE_VALUE; + VGMEMP_CREATE(e,0,0); *env = e; return MDB_SUCCESS; } @@ -2827,8 +2855,10 @@ mdb_env_close(MDB_env *env) if (env == NULL) return; + VGMEMP_DESTROY(env); while (env->me_dpages) { dp = env->me_dpages; + VALGRIND_MAKE_MEM_DEFINED(&dp->mp_next, sizeof(dp->mp_next)); env->me_dpages = dp->mp_next; free(dp); } @@ -5635,6 +5665,7 @@ newsep: /* return tmp page to freelist */ copy->mp_next = mc->mc_txn->mt_env->me_dpages; + VGMEMP_FREE(mc->mc_txn->mt_env, copy); mc->mc_txn->mt_env->me_dpages = copy; done: {