From 822b2d502fd5383dd60497097dff94fe8a2ccb53 Mon Sep 17 00:00:00 2001 From: Hallvard Furuseth Date: Tue, 19 May 2015 20:33:35 +0200 Subject: [PATCH 1/7] ITS#8117 Fix MDB_INTEGERDUP keysize doc + md_dcmp --- libraries/liblmdb/lmdb.h | 14 +++++++------- libraries/liblmdb/mdb.c | 27 ++++++++++++++++++--------- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h index 871f07d1e7..e8aed94418 100644 --- a/libraries/liblmdb/lmdb.h +++ b/libraries/liblmdb/lmdb.h @@ -296,12 +296,12 @@ typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *rel #define MDB_REVERSEKEY 0x02 /** use sorted duplicates */ #define MDB_DUPSORT 0x04 - /** numeric keys in native byte order. + /** numeric keys in native byte order: either unsigned int or size_t. * The keys must all be of the same size. */ #define MDB_INTEGERKEY 0x08 /** with #MDB_DUPSORT, sorted dup items have fixed size */ #define MDB_DUPFIXED 0x10 - /** with #MDB_DUPSORT, dups are numeric in native byte order */ + /** with #MDB_DUPSORT, dups are #MDB_INTEGERKEY-style integers */ #define MDB_INTEGERDUP 0x20 /** with #MDB_DUPSORT, use reverse string dups */ #define MDB_REVERSEDUP 0x40 @@ -1052,9 +1052,9 @@ int mdb_txn_renew(MDB_txn *txn); * keys may have multiple data items, stored in sorted order.) By default * keys must be unique and may have only a single data item. *
  • #MDB_INTEGERKEY - * Keys are binary integers in native byte order. Setting this option - * requires all keys to be the same size, typically sizeof(int) - * or sizeof(size_t). + * Keys are binary integers in native byte order, either unsigned int + * or size_t, and will be sorted as such. + * The keys must all be of the same size. *
  • #MDB_DUPFIXED * This flag may only be used in combination with #MDB_DUPSORT. This option * tells the library that the data items for this database are all the same @@ -1062,8 +1062,8 @@ int mdb_txn_renew(MDB_txn *txn); * all data items are the same size, the #MDB_GET_MULTIPLE and #MDB_NEXT_MULTIPLE * cursor operations may be used to retrieve multiple items at once. *
  • #MDB_INTEGERDUP - * This option specifies that duplicate data items are also integers, and - * should be sorted as such. + * This option specifies that duplicate data items are binary integers, + * similar to #MDB_INTEGERKEY keys. *
  • #MDB_REVERSEDUP * This option specifies that duplicate data items should be compared as * strings in reverse order. diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 7439dd015b..86635bfea6 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -1245,6 +1245,13 @@ static void mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi); static MDB_cmp_func mdb_cmp_memn, mdb_cmp_memnr, mdb_cmp_int, mdb_cmp_cint, mdb_cmp_long; /** @endcond */ +/** Compare two items pointing at size_t's of unknown alignment. */ +#ifdef MISALIGNED_OK +# define mdb_cmp_clong mdb_cmp_long +#else +# define mdb_cmp_clong mdb_cmp_cint +#endif + #ifdef _WIN32 static SECURITY_DESCRIPTOR mdb_null_sd; static SECURITY_ATTRIBUTES mdb_all_sa; @@ -1558,7 +1565,12 @@ mdb_cmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b) int mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b) { - return txn->mt_dbxs[dbi].md_dcmp(a, b); + MDB_cmp_func *dcmp = txn->mt_dbxs[dbi].md_dcmp; +#if UINT_MAX < SIZE_MAX + if (dcmp == mdb_cmp_int && a->mv_size == sizeof(size_t)) + dcmp = mdb_cmp_clong; +#endif + return dcmp(a, b); } /** Allocate memory for a page. @@ -4772,7 +4784,11 @@ mdb_cmp_long(const MDB_val *a, const MDB_val *b) *(size_t *)a->mv_data > *(size_t *)b->mv_data; } -/** Compare two items pointing at aligned unsigned int's */ +/** Compare two items pointing at aligned unsigned int's. + * + * This is also set as #MDB_INTEGERDUP|#MDB_DUPFIXED's #MDB_dbx.%md_dcmp, + * but #mdb_cmp_clong() is called instead if the data type is size_t. + */ static int mdb_cmp_int(const MDB_val *a, const MDB_val *b) { @@ -4810,13 +4826,6 @@ mdb_cmp_cint(const MDB_val *a, const MDB_val *b) #endif } -/** Compare two items pointing at size_t's of unknown alignment. */ -#ifdef MISALIGNED_OK -# define mdb_cmp_clong mdb_cmp_long -#else -# define mdb_cmp_clong mdb_cmp_cint -#endif - /** Compare two items lexically */ static int mdb_cmp_memn(const MDB_val *a, const MDB_val *b) From 91743d66e6e91058fff8917b5e39b8d49bfa4d7a Mon Sep 17 00:00:00 2001 From: Hallvard Furuseth Date: Wed, 20 May 2015 03:17:47 +0200 Subject: [PATCH 2/7] mtest*.c: Fix MDB_NOOVERWRITE, plug cursor leak --- libraries/liblmdb/mtest.c | 6 ++++-- libraries/liblmdb/mtest2.c | 4 ++-- libraries/liblmdb/mtest6.c | 9 ++++++--- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/libraries/liblmdb/mtest.c b/libraries/liblmdb/mtest.c index 7efa8b59e1..9d15088b0c 100644 --- a/libraries/liblmdb/mtest.c +++ b/libraries/liblmdb/mtest.c @@ -54,12 +54,13 @@ int main(int argc,char * argv[]) key.mv_size = sizeof(int); key.mv_data = sval; - data.mv_size = sizeof(sval); - data.mv_data = sval; printf("Adding %d values\n", count); for (i=0;i in each iteration, since MDB_NOOVERWRITE may modify it */ + data.mv_size = sizeof(sval); + data.mv_data = sval; if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) { j++; data.mv_size = sizeof(sval); @@ -130,6 +131,7 @@ int main(int argc,char * argv[]) (int) key.mv_size, (char *) key.mv_data, (int) data.mv_size, (char *) data.mv_data); + mdb_cursor_close(cursor); mdb_txn_abort(txn); printf("Deleting with cursor\n"); diff --git a/libraries/liblmdb/mtest2.c b/libraries/liblmdb/mtest2.c index cc6ecf6026..eacbe59d53 100644 --- a/libraries/liblmdb/mtest2.c +++ b/libraries/liblmdb/mtest2.c @@ -57,12 +57,12 @@ int main(int argc,char * argv[]) key.mv_size = sizeof(int); key.mv_data = sval; - data.mv_size = sizeof(sval); - data.mv_data = sval; printf("Adding %d values\n", count); for (i=0;i Date: Wed, 27 May 2015 22:26:54 +0200 Subject: [PATCH 3/7] ITS#8157 mdb_txn_renew0(): init after error checks --- libraries/liblmdb/mdb.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 86635bfea6..71852b1f6e 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -2498,15 +2498,11 @@ mdb_txn_renew0(MDB_txn *txn) MDB_env *env = txn->mt_env; MDB_txninfo *ti = env->me_txns; MDB_meta *meta; - unsigned int i, nr; + unsigned int i, nr, flags = txn->mt_flags; uint16_t x; int rc, new_notls = 0; - if (txn->mt_flags & MDB_TXN_RDONLY) { - txn->mt_flags = MDB_TXN_RDONLY; - /* Setup db info */ - txn->mt_numdbs = env->me_numdbs; - txn->mt_dbxs = env->me_dbxs; /* mostly static anyway */ + if ((flags &= MDB_TXN_RDONLY) != 0) { if (!ti) { meta = env->me_metas[ mdb_env_pick_meta(env) ]; txn->mt_txnid = meta->mm_txnid; @@ -2559,6 +2555,7 @@ mdb_txn_renew0(MDB_txn *txn) txn->mt_u.reader = r; meta = env->me_metas[txn->mt_txnid & 1]; } + txn->mt_dbxs = env->me_dbxs; /* mostly static anyway */ } else { if (ti) { LOCK_MUTEX_W(env); @@ -2569,14 +2566,11 @@ mdb_txn_renew0(MDB_txn *txn) meta = env->me_metas[ mdb_env_pick_meta(env) ]; txn->mt_txnid = meta->mm_txnid; } - /* Setup db info */ - txn->mt_numdbs = env->me_numdbs; txn->mt_txnid++; #if MDB_DEBUG if (txn->mt_txnid == mdb_debug_start) mdb_debug = 1; #endif - txn->mt_flags = 0; txn->mt_child = NULL; txn->mt_loose_pgs = NULL; txn->mt_loose_count = 0; @@ -2596,6 +2590,10 @@ mdb_txn_renew0(MDB_txn *txn) /* Moved to here to avoid a data race in read TXNs */ txn->mt_next_pgno = meta->mm_last_pg+1; + txn->mt_flags = flags; + + /* Setup db info */ + txn->mt_numdbs = env->me_numdbs; for (i=2; imt_numdbs; i++) { x = env->me_dbflags[i]; txn->mt_dbs[i].md_flags = x & PERSISTENT_FLAGS; From 277e536ffa7090ff334254e6eaf2c8ead1cdbd53 Mon Sep 17 00:00:00 2001 From: Hallvard Furuseth Date: Thu, 28 May 2015 00:12:53 +0200 Subject: [PATCH 4/7] ITS#8157 --- libraries/liblmdb/CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/liblmdb/CHANGES b/libraries/liblmdb/CHANGES index 6d32dc9daf..51a528eb87 100644 --- a/libraries/liblmdb/CHANGES +++ b/libraries/liblmdb/CHANGES @@ -8,6 +8,7 @@ LMDB 0.9.15 Release Engineering Fix mdb_load with large values (ITS#8066) Fix to retry writes on EINTR (ITS#8106) Fix mdb_cursor_del on empty DB (ITS#8109) + Fix mdb_txn_renew after error (ITS#8157) Fix MDB_INTEGERDUP key compare (ITS#8117) Added workaround for fdatasync bug in ext3fs Build From a0b96697c5ac66c7099028909cc9423be3fa68ce Mon Sep 17 00:00:00 2001 From: Hallvard Furuseth Date: Thu, 28 May 2015 20:51:24 +0200 Subject: [PATCH 5/7] ITS#8156 Fix MDB_MAXKEYSIZE doc --- libraries/liblmdb/mdb.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 71852b1f6e..6bdf3151dc 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -446,12 +446,17 @@ static txnid_t mdb_debug_start; /** The version number for a database's lockfile format. */ #define MDB_LOCK_VERSION 1 - /** @brief The max size of a key we can write, or 0 for dynamic max. + /** @brief The max size of a key we can write, or 0 for computed max. * - * Define this as 0 to compute the max from the page size. 511 - * is default for backwards compat: liblmdb <= 0.9.10 can break - * when modifying a DB with keys/dupsort data bigger than its max. - * #MDB_DEVEL sets the default to 0. + * This macro should normally be left alone or set to 0. + * Note that a database with big keys or dupsort data cannot be + * reliably modified by a liblmdb which uses a smaller max. + * The default is 511 for backwards compat, or 0 when #MDB_DEVEL. + * + * Other values are allowed, for backwards compat. However: + * A value bigger than the computed max can break if you do not + * know what you are doing, and liblmdb <= 0.9.10 can break when + * modifying a DB with keys/dupsort data bigger than its max. * * Data items in an #MDB_DUPSORT database are also limited to * this size, since they're actually keys of a sub-DB. Keys and From 2a1dd5ae1cc8addffd1bcf348e87eab1a40f69b2 Mon Sep 17 00:00:00 2001 From: Hallvard Furuseth Date: Thu, 28 May 2015 21:13:24 +0200 Subject: [PATCH 6/7] ITS#8156 --- libraries/liblmdb/CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/liblmdb/CHANGES b/libraries/liblmdb/CHANGES index 51a528eb87..fdad64696e 100644 --- a/libraries/liblmdb/CHANGES +++ b/libraries/liblmdb/CHANGES @@ -18,6 +18,7 @@ LMDB 0.9.15 Release Engineering Misc. Android/Windows cleanup Documentation Fix MDB_APPEND doc + Fix MDB_MAXKEYSIZE doc (ITS#8156) Clarify mdb_dbi_open doc LMDB 0.9.14 Release (2014/09/20) From 249d2b84dc74cb3d6e9978bfb3ea63a5c325056f Mon Sep 17 00:00:00 2001 From: Hallvard Furuseth Date: Fri, 29 May 2015 03:08:32 +0200 Subject: [PATCH 7/7] more CHANGES --- libraries/liblmdb/CHANGES | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/libraries/liblmdb/CHANGES b/libraries/liblmdb/CHANGES index fdad64696e..710b4772c1 100644 --- a/libraries/liblmdb/CHANGES +++ b/libraries/liblmdb/CHANGES @@ -4,21 +4,27 @@ LMDB 0.9.15 Release Engineering Fix txn init (ITS#7961,#7987) Fix MDB_PREV_DUP (ITS#7955,#7671) Fix compact of empty env (ITS#7956) + Fix mdb_copy file mode + Fix mdb_env_close() after failed mdb_env_open() Fix mdb_rebalance collapsing root (ITS#8062) Fix mdb_load with large values (ITS#8066) Fix to retry writes on EINTR (ITS#8106) Fix mdb_cursor_del on empty DB (ITS#8109) - Fix mdb_txn_renew after error (ITS#8157) Fix MDB_INTEGERDUP key compare (ITS#8117) + Fix error handling (ITS#7959,#8157,etc.) + Fix race conditions (ITS#7969,7970) Added workaround for fdatasync bug in ext3fs Build Don't use -fPIC for static lib Update .gitignore (ITS#7952,#7953) - Cleanup for "make test" (ITS#7841) + Cleanup for "make test" (ITS#7841), "make clean", mtest*.c Misc. Android/Windows cleanup Documentation Fix MDB_APPEND doc Fix MDB_MAXKEYSIZE doc (ITS#8156) + Fix mdb_cursor_put,mdb_cursor_del EACCES description + Fix mdb_env_sync(MDB_RDONLY env) doc (ITS#8021) + Clarify MDB_WRITEMAP doc (ITS#8021) Clarify mdb_dbi_open doc LMDB 0.9.14 Release (2014/09/20)