Merge remote-tracking branch 'origin/mdb.RE/0.9'

libraries/liblmdb/CHANGES

@@ -4,19 +4,27 @@ LMDB 0.9.15 Release Engineering
 	Fix txn init (ITS#7961,#7987)
 	Fix MDB_PREV_DUP (ITS#7955,#7671)
 	Fix compact of empty env (ITS#7956)
+	Fix mdb_copy file mode
+	Fix mdb_env_close() after failed mdb_env_open()
 	Fix mdb_rebalance collapsing root (ITS#8062)
 	Fix mdb_load with large values (ITS#8066)
 	Fix to retry writes on EINTR (ITS#8106)
 	Fix mdb_cursor_del on empty DB (ITS#8109)
 	Fix MDB_INTEGERDUP key compare (ITS#8117)
+	Fix error handling (ITS#7959,#8157,etc.)
+	Fix race conditions (ITS#7969,7970)
 	Added workaround for fdatasync bug in ext3fs
 	Build
 		Don't use -fPIC for static lib
 		Update .gitignore (ITS#7952,#7953)
-		Cleanup for "make test" (ITS#7841)
+		Cleanup for "make test" (ITS#7841), "make clean", mtest*.c
 		Misc. Android/Windows cleanup
 	Documentation
 		Fix MDB_APPEND doc
+		Fix MDB_MAXKEYSIZE doc (ITS#8156)
+		Fix mdb_cursor_put,mdb_cursor_del EACCES description
+		Fix mdb_env_sync(MDB_RDONLY env) doc (ITS#8021)
+		Clarify MDB_WRITEMAP doc (ITS#8021)
 		Clarify mdb_dbi_open doc
 
 LMDB 0.9.14 Release (2014/09/20)

libraries/liblmdb/lmdb.h

@@ -296,12 +296,12 @@ typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *rel
 #define MDB_REVERSEKEY	0x02
 	/** use sorted duplicates */
 #define MDB_DUPSORT		0x04
-	/** numeric keys in native byte order.
+	/** numeric keys in native byte order: either unsigned int or size_t.
 	 *  The keys must all be of the same size. */
 #define MDB_INTEGERKEY	0x08
 	/** with #MDB_DUPSORT, sorted dup items have fixed size */
 #define MDB_DUPFIXED	0x10
-	/** with #MDB_DUPSORT, dups are numeric in native byte order */
+	/** with #MDB_DUPSORT, dups are #MDB_INTEGERKEY-style integers */
 #define MDB_INTEGERDUP	0x20
 	/** with #MDB_DUPSORT, use reverse string dups */
 #define MDB_REVERSEDUP	0x40
@@ -1052,9 +1052,9 @@ int  mdb_txn_renew(MDB_txn *txn);
 	 *		keys may have multiple data items, stored in sorted order.) By default
 	 *		keys must be unique and may have only a single data item.
 	 *	<li>#MDB_INTEGERKEY
-	 *		Keys are binary integers in native byte order. Setting this option
-	 *		requires all keys to be the same size, typically sizeof(int)
-	 *		or sizeof(size_t).
+	 *		Keys are binary integers in native byte order, either unsigned int
+	 *		or size_t, and will be sorted as such.
+	 *		The keys must all be of the same size.
 	 *	<li>#MDB_DUPFIXED
 	 *		This flag may only be used in combination with #MDB_DUPSORT. This option
 	 *		tells the library that the data items for this database are all the same
@@ -1062,8 +1062,8 @@ int  mdb_txn_renew(MDB_txn *txn);
 	 *		all data items are the same size, the #MDB_GET_MULTIPLE and #MDB_NEXT_MULTIPLE
 	 *		cursor operations may be used to retrieve multiple items at once.
 	 *	<li>#MDB_INTEGERDUP
-	 *		This option specifies that duplicate data items are also integers, and
-	 *		should be sorted as such.
+	 *		This option specifies that duplicate data items are binary integers,
+	 *		similar to #MDB_INTEGERKEY keys.
 	 *	<li>#MDB_REVERSEDUP
 	 *		This option specifies that duplicate data items should be compared as
 	 *		strings in reverse order.

libraries/liblmdb/mdb.c

@@ -446,12 +446,17 @@ static txnid_t mdb_debug_start;
 	/**	The version number for a database's lockfile format. */
 #define MDB_LOCK_VERSION	 1
 
-	/**	@brief The max size of a key we can write, or 0 for dynamic max.
+	/**	@brief The max size of a key we can write, or 0 for computed max.
 	 *
-	 *	Define this as 0 to compute the max from the page size.  511
-	 *	is default for backwards compat: liblmdb <= 0.9.10 can break
-	 *	when modifying a DB with keys/dupsort data bigger than its max.
-	 *	#MDB_DEVEL sets the default to 0.
+	 *	This macro should normally be left alone or set to 0.
+	 *	Note that a database with big keys or dupsort data cannot be
+	 *	reliably modified by a liblmdb which uses a smaller max.
+	 *	The default is 511 for backwards compat, or 0 when #MDB_DEVEL.
+	 *
+	 *	Other values are allowed, for backwards compat.  However:
+	 *	A value bigger than the computed max can break if you do not
+	 *	know what you are doing, and liblmdb <= 0.9.10 can break when
+	 *	modifying a DB with keys/dupsort data bigger than its max.
 	 *
 	 *	Data items in an #MDB_DUPSORT database are also limited to
 	 *	this size, since they're actually keys of a sub-DB.  Keys and
@@ -1245,6 +1250,13 @@ static void mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi);
 static MDB_cmp_func	mdb_cmp_memn, mdb_cmp_memnr, mdb_cmp_int, mdb_cmp_cint, mdb_cmp_long;
 /** @endcond */
 
+/** Compare two items pointing at size_t's of unknown alignment. */
+#ifdef MISALIGNED_OK
+# define mdb_cmp_clong mdb_cmp_long
+#else
+# define mdb_cmp_clong mdb_cmp_cint
+#endif
+
 #ifdef _WIN32
 static SECURITY_DESCRIPTOR mdb_null_sd;
 static SECURITY_ATTRIBUTES mdb_all_sa;
@@ -1558,7 +1570,12 @@ mdb_cmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b)
 int
 mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b)
 {
-	return txn->mt_dbxs[dbi].md_dcmp(a, b);
+	MDB_cmp_func *dcmp = txn->mt_dbxs[dbi].md_dcmp;
+#if UINT_MAX < SIZE_MAX
+	if (dcmp == mdb_cmp_int && a->mv_size == sizeof(size_t))
+		dcmp = mdb_cmp_clong;
+#endif
+	return dcmp(a, b);
 }
 
 /** Allocate memory for a page.
@@ -2486,15 +2503,11 @@ mdb_txn_renew0(MDB_txn *txn)
 	MDB_env *env = txn->mt_env;
 	MDB_txninfo *ti = env->me_txns;
 	MDB_meta *meta;
-	unsigned int i, nr;
+	unsigned int i, nr, flags = txn->mt_flags;
 	uint16_t x;
 	int rc, new_notls = 0;
 
-	if (txn->mt_flags & MDB_TXN_RDONLY) {
-		txn->mt_flags = MDB_TXN_RDONLY;
-		/* Setup db info */
-		txn->mt_numdbs = env->me_numdbs;
-		txn->mt_dbxs = env->me_dbxs;	/* mostly static anyway */
+	if ((flags &= MDB_TXN_RDONLY) != 0) {
 		if (!ti) {
 			meta = env->me_metas[ mdb_env_pick_meta(env) ];
 			txn->mt_txnid = meta->mm_txnid;
@@ -2547,6 +2560,7 @@ mdb_txn_renew0(MDB_txn *txn)
 			txn->mt_u.reader = r;
 			meta = env->me_metas[txn->mt_txnid & 1];
 		}
+		txn->mt_dbxs = env->me_dbxs;	/* mostly static anyway */
 	} else {
 		if (ti) {
 			LOCK_MUTEX_W(env);
@@ -2557,14 +2571,11 @@ mdb_txn_renew0(MDB_txn *txn)
 			meta = env->me_metas[ mdb_env_pick_meta(env) ];
 			txn->mt_txnid = meta->mm_txnid;
 		}
-		/* Setup db info */
-		txn->mt_numdbs = env->me_numdbs;
 		txn->mt_txnid++;
 #if MDB_DEBUG
 		if (txn->mt_txnid == mdb_debug_start)
 			mdb_debug = 1;
 #endif
-		txn->mt_flags = 0;
 		txn->mt_child = NULL;
 		txn->mt_loose_pgs = NULL;
 		txn->mt_loose_count = 0;
@@ -2584,6 +2595,10 @@ mdb_txn_renew0(MDB_txn *txn)
 	/* Moved to here to avoid a data race in read TXNs */
 	txn->mt_next_pgno = meta->mm_last_pg+1;
 
+	txn->mt_flags = flags;
+
+	/* Setup db info */
+	txn->mt_numdbs = env->me_numdbs;
 	for (i=2; i<txn->mt_numdbs; i++) {
 		x = env->me_dbflags[i];
 		txn->mt_dbs[i].md_flags = x & PERSISTENT_FLAGS;
@@ -4772,7 +4787,11 @@ mdb_cmp_long(const MDB_val *a, const MDB_val *b)
 		*(size_t *)a->mv_data > *(size_t *)b->mv_data;
 }
 
-/** Compare two items pointing at aligned unsigned int's */
+/** Compare two items pointing at aligned unsigned int's.
+ *
+ *	This is also set as #MDB_INTEGERDUP|#MDB_DUPFIXED's #MDB_dbx.%md_dcmp,
+ *	but #mdb_cmp_clong() is called instead if the data type is size_t.
+ */
 static int
 mdb_cmp_int(const MDB_val *a, const MDB_val *b)
 {
@@ -4810,13 +4829,6 @@ mdb_cmp_cint(const MDB_val *a, const MDB_val *b)
 #endif
 }
 
-/** Compare two items pointing at size_t's of unknown alignment. */
-#ifdef MISALIGNED_OK
-# define mdb_cmp_clong mdb_cmp_long
-#else
-# define mdb_cmp_clong mdb_cmp_cint
-#endif
-
 /** Compare two items lexically */
 static int
 mdb_cmp_memn(const MDB_val *a, const MDB_val *b)

libraries/liblmdb/mtest.c

@@ -54,12 +54,13 @@ int main(int argc,char * argv[])
    
 		key.mv_size = sizeof(int);
 		key.mv_data = sval;
-		data.mv_size = sizeof(sval);
-		data.mv_data = sval;
 
 		printf("Adding %d values\n", count);
 	    for (i=0;i<count;i++) {	
 			sprintf(sval, "%03x %d foo bar", values[i], values[i]);
+			/* Set <data> in each iteration, since MDB_NOOVERWRITE may modify it */
+			data.mv_size = sizeof(sval);
+			data.mv_data = sval;
 			if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) {
 				j++;
 				data.mv_size = sizeof(sval);
@@ -130,6 +131,7 @@ int main(int argc,char * argv[])
 				(int) key.mv_size,  (char *) key.mv_data,
 				(int) data.mv_size, (char *) data.mv_data);
 
+		mdb_cursor_close(cursor);
 		mdb_txn_abort(txn);
 
 		printf("Deleting with cursor\n");

libraries/liblmdb/mtest2.c

@@ -57,12 +57,12 @@ int main(int argc,char * argv[])
    
 	key.mv_size = sizeof(int);
 	key.mv_data = sval;
-	data.mv_size = sizeof(sval);
-	data.mv_data = sval;
 
 	printf("Adding %d values\n", count);
 	for (i=0;i<count;i++) {	
 		sprintf(sval, "%03x %d foo bar", values[i], values[i]);
+		data.mv_size = sizeof(sval);
+		data.mv_data = sval;
 		if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE)))
 			j++;
 	}

libraries/liblmdb/mtest6.c

@@ -31,7 +31,7 @@ int main(int argc,char * argv[])
 	int i = 0, j = 0, rc;
 	MDB_env *env;
 	MDB_dbi dbi;
-	MDB_val key, data;
+	MDB_val key, data, sdata;
 	MDB_txn *txn;
 	MDB_stat mst;
 	MDB_cursor *cursor;
@@ -55,25 +55,28 @@ int main(int argc,char * argv[])
 	sval = calloc(1, mst.ms_psize / 4);
 	key.mv_size = sizeof(long);
 	key.mv_data = &kval;
-	data.mv_size = mst.ms_psize / 4 - 30;
-	data.mv_data = sval;
+	sdata.mv_size = mst.ms_psize / 4 - 30;
+	sdata.mv_data = sval;
 
 	printf("Adding 12 values, should yield 3 splits\n");
 	for (i=0;i<12;i++) {
 		kval = i*5;
 		sprintf(sval, "%08x", kval);
+		data = sdata;
 		(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
 	}
 	printf("Adding 12 more values, should yield 3 splits\n");
 	for (i=0;i<12;i++) {
 		kval = i*5+4;
 		sprintf(sval, "%08x", kval);
+		data = sdata;
 		(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
 	}
 	printf("Adding 12 more values, should yield 3 splits\n");
 	for (i=0;i<12;i++) {
 		kval = i*5+1;
 		sprintf(sval, "%08x", kval);
+		data = sdata;
 		(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
 	}
 	E(mdb_cursor_get(cursor, &key, &data, MDB_FIRST));