From a2ac10107e2fb845c4a38a339239063ec4407d84 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Wed, 16 Jul 2014 07:15:58 -0700
Subject: [PATCH 001/504] Add MDB_ROBUST

---
 libraries/liblmdb/lmdb.h |  16 ++-
 libraries/liblmdb/mdb.c  | 230 +++++++++++++++++++++++++++++----------
 2 files changed, 184 insertions(+), 62 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index bdbb0b909b..82a5410ab1 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -49,7 +49,9 @@
  *	  stale locks can block further operation.
  *
  *	  Fix: Check for stale readers periodically, using the
- *	  #mdb_reader_check function or the \ref mdb_stat_1 "mdb_stat" tool. Or just
+ *	  #mdb_reader_check function or the \ref mdb_stat_1 "mdb_stat" tool.
+ *	  Catch stale
+ *	  locks with option MDB_ROBUST if supported (non-BSD). Or just
  *	  make all programs using the database close it; the lockfile
  *	  is always reset on first open of the environment.
  *
@@ -105,6 +107,7 @@
  *	  The transaction becomes "long-lived" as above until a check
  *	  for stale readers is performed or the lockfile is reset,
  *	  since the process may not remove it from the lockfile.
+ *	  Except write-transactions on Unix with MDB_ROBUST or on Windows.
  *
  *	- If you do that anyway, do a periodic check for stale readers. Or
  *	  close the environment once in a while, so the lockfile can get reset.
@@ -287,6 +290,8 @@ typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *rel
 #define MDB_NORDAHEAD	0x800000
 	/** don't initialize malloc'd memory before writing to datafile */
 #define MDB_NOMEMINIT	0x1000000
+	/** catch stale locks if supported (not on BSD, needs robust mutexes) */
+#define MDB_ROBUST		0x2000000
 /** @} */
 
 /**	@defgroup	mdb_dbi_open	Database Flags
@@ -391,7 +396,7 @@ typedef enum MDB_cursor_op {
 #define MDB_PAGE_NOTFOUND	(-30797)
 	/** Located page was wrong type */
 #define MDB_CORRUPTED	(-30796)
-	/** Update of meta page failed, probably I/O error */
+	/** Update of meta page failed or environment had fatal error */
 #define MDB_PANIC		(-30795)
 	/** Environment version mismatch */
 #define MDB_VERSION_MISMATCH	(-30794)
@@ -511,6 +516,12 @@ int  mdb_env_create(MDB_env **env);
 	 *		Open the environment in read-only mode. No write operations will be
 	 *		allowed. LMDB will still modify the lock file - except on read-only
 	 *		filesystems, where LMDB does not use locks.
+	 *	<li>#MDB_ROBUST
+	 *		Initialize the lockfile to catch stale locks if robust mutexes
+	 *		are supported, so aborted processes will not block others.
+	 *		Ignored when another process has the environment open. Unsupported
+	 *		by liblmdb built with MDB_USE_POSIX_SEM (such as BSD systems).
+	 *		Enabled by default on Windows. Some locking slowdown on Unix.
 	 *	<li>#MDB_WRITEMAP
 	 *		Use a writeable memory map unless MDB_RDONLY is set. This is faster
 	 *		and uses fewer mallocs, but loses protection from application bugs
@@ -727,6 +738,7 @@ void mdb_env_close(MDB_env *env);
 	 * This may be used to set some flags in addition to those from
 	 * #mdb_env_open(), or to unset these flags.  If several threads
 	 * change the flags at the same time, the result is undefined.
+	 * Most flags cannot be changed after #mdb_env_open().
 	 * @param[in] env An environment handle returned by #mdb_env_create()
 	 * @param[in] flags The flags to change, bitwise OR'ed together
 	 * @param[in] onoff A non-zero value sets the flags, zero clears them.
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 60dfd8db3e..70c4de263a 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -191,6 +191,10 @@ extern int cacheflush(char *addr, int nbytes, int cache);
 	/** Features under development */
 #ifndef MDB_DEVEL
 #define MDB_DEVEL 0
+#endif
+
+#if MDB_DEVEL && (defined(_WIN32) || (defined(EOWNERDEAD) && !defined(MDB_USE_POSIX_SEM)))
+#define MDB_ROBUST_SUPPORTED	1
 #endif
 
 	/** Wrapper around __func__, which is a C99 feature */
@@ -210,6 +214,7 @@ extern int cacheflush(char *addr, int nbytes, int cache);
 #define pthread_t	HANDLE
 #define pthread_mutex_t	HANDLE
 #define pthread_cond_t	HANDLE
+typedef HANDLE mdb_mutex_t;
 #define pthread_key_t	DWORD
 #define pthread_self()	GetCurrentThreadId()
 #define pthread_key_create(x,y)	\
@@ -217,16 +222,16 @@ extern int cacheflush(char *addr, int nbytes, int cache);
 #define pthread_key_delete(x)	TlsFree(x)
 #define pthread_getspecific(x)	TlsGetValue(x)
 #define pthread_setspecific(x,y)	(TlsSetValue(x,y) ? 0 : ErrCode())
+#define pthread_mutex_consistent(mutex) 0
 #define pthread_mutex_unlock(x)	ReleaseMutex(*x)
 #define pthread_mutex_lock(x)	WaitForSingleObject(*x, INFINITE)
 #define pthread_cond_signal(x)	SetEvent(*x)
 #define pthread_cond_wait(cond,mutex)	do{SignalObjectAndWait(*mutex, *cond, INFINITE, FALSE); WaitForSingleObject(*mutex, INFINITE);}while(0)
 #define THREAD_CREATE(thr,start,arg)	thr=CreateThread(NULL,0,start,arg,0,NULL)
 #define THREAD_FINISH(thr)	WaitForSingleObject(thr, INFINITE)
-#define LOCK_MUTEX_R(env)	pthread_mutex_lock(&(env)->me_rmutex)
-#define UNLOCK_MUTEX_R(env)	pthread_mutex_unlock(&(env)->me_rmutex)
-#define LOCK_MUTEX_W(env)	pthread_mutex_lock(&(env)->me_wmutex)
-#define UNLOCK_MUTEX_W(env)	pthread_mutex_unlock(&(env)->me_wmutex)
+#define MDB_MUTEX(env, rw)		((env)->me_##rw##mutex)
+#define LOCK_MUTEX0(mutex)		WaitForSingleObject(mutex, INFINITE)
+#define UNLOCK_MUTEX(mutex)		ReleaseMutex(mutex)
 #define getpid()	GetCurrentProcessId()
 #define	MDB_FDATASYNC(fd)	(!FlushFileBuffers(fd))
 #define	MDB_MSYNC(addr,len,flags)	(!FlushViewOfFile(addr,len))
@@ -251,10 +256,10 @@ extern int cacheflush(char *addr, int nbytes, int cache);
 
 #ifdef MDB_USE_POSIX_SEM
 
-#define LOCK_MUTEX_R(env)	mdb_sem_wait((env)->me_rmutex)
-#define UNLOCK_MUTEX_R(env)	sem_post((env)->me_rmutex)
-#define LOCK_MUTEX_W(env)	mdb_sem_wait((env)->me_wmutex)
-#define UNLOCK_MUTEX_W(env)	sem_post((env)->me_wmutex)
+typedef sem_t *mdb_mutex_t;
+#define MDB_MUTEX(env, rw)		((env)->me_##rw##mutex)
+#define LOCK_MUTEX0(mutex)		mdb_sem_wait(mutex)
+#define UNLOCK_MUTEX(mutex)		sem_post(mutex)
 
 static int
 mdb_sem_wait(sem_t *sem)
@@ -265,21 +270,19 @@ mdb_sem_wait(sem_t *sem)
 }
 
 #else
-	/** Lock the reader mutex.
+	/** Pointer/HANDLE type of shared mutex/semaphore.
 	 */
-#define LOCK_MUTEX_R(env)	pthread_mutex_lock(&(env)->me_txns->mti_mutex)
-	/** Unlock the reader mutex.
+typedef pthread_mutex_t *mdb_mutex_t;
+	/** Mutex for the reader table (rw = r) or write transaction (rw = w).
 	 */
-#define UNLOCK_MUTEX_R(env)	pthread_mutex_unlock(&(env)->me_txns->mti_mutex)
-
-	/** Lock the writer mutex.
-	 *	Only a single write transaction is allowed at a time. Other writers
-	 *	will block waiting for this mutex.
+#define MDB_MUTEX(env, rw)	(&(env)->me_txns->mti_##rw##mutex)
+	/** Lock the reader or writer mutex.
+	 *	Returns 0 or a code to give #mdb_mutex_failed(), as in #LOCK_MUTEX().
 	 */
-#define LOCK_MUTEX_W(env)	pthread_mutex_lock(&(env)->me_txns->mti_wmutex)
-	/** Unlock the writer mutex.
+#define LOCK_MUTEX0(mutex)	pthread_mutex_lock(mutex)
+	/** Unlock the reader or writer mutex.
 	 */
-#define UNLOCK_MUTEX_W(env)	pthread_mutex_unlock(&(env)->me_txns->mti_wmutex)
+#define UNLOCK_MUTEX(mutex)	pthread_mutex_unlock(mutex)
 #endif	/* MDB_USE_POSIX_SEM */
 
 	/** Get the error code for the last failed system function.
@@ -313,6 +316,19 @@ mdb_sem_wait(sem_t *sem)
 
 /** @} */
 
+#ifdef MDB_ROBUST_SUPPORTED
+	/** Lock mutex, handle any error, set rc = result.
+	 *	Return 0 on success, nonzero (not rc) on error.
+	 */
+#define LOCK_MUTEX(rc, env, mutex) \
+	(((rc) = LOCK_MUTEX0(mutex)) && \
+	 ((rc) = mdb_mutex_failed(env, mutex, rc)))
+static int mdb_mutex_failed(MDB_env *env, mdb_mutex_t mutex, int rc);
+#else
+#define LOCK_MUTEX(rc, env, mutex) ((rc) = LOCK_MUTEX0(mutex))
+#define mdb_mutex_failed(env, mutex, rc) (rc)
+#endif
+
 #ifndef _WIN32
 /**	A flag for opening a file and requesting synchronous data writes.
  *	This is only used when writing a meta page. It's not strictly needed;
@@ -436,7 +452,7 @@ static txnid_t mdb_debug_start;
 	/**	The version number for a database's datafile format. */
 #define MDB_DATA_VERSION	 ((MDB_DEVEL) ? 999 : 1)
 	/**	The version number for a database's lockfile format. */
-#define MDB_LOCK_VERSION	 1
+#define MDB_LOCK_VERSION	 ((MDB_DEVEL) ? 999 : 1)
 
 	/**	@brief The max size of a key we can write, or 0 for dynamic max.
 	 *
@@ -624,9 +640,9 @@ typedef struct MDB_txbody {
 	char	mtb_rmname[MNAME_LEN];
 #else
 		/** Mutex protecting access to this table.
-		 *	This is the reader lock that #LOCK_MUTEX_R acquires.
+		 *	This is the #MDB_MUTEX(env,r) reader table lock.
 		 */
-	pthread_mutex_t	mtb_mutex;
+	pthread_mutex_t	mtb_rmutex;
 #endif
 		/**	The ID of the last transaction committed to the database.
 		 *	This is recorded here only for convenience; the value can always
@@ -638,6 +654,8 @@ typedef struct MDB_txbody {
 		 *	when readers release their slots.
 		 */
 	unsigned	mtb_numreaders;
+		/** Flags which the lock file was initialized with. */
+	unsigned	mtb_flags;
 } MDB_txbody;
 
 	/** The actual reader table definition. */
@@ -646,10 +664,11 @@ typedef struct MDB_txninfo {
 		MDB_txbody mtb;
 #define mti_magic	mt1.mtb.mtb_magic
 #define mti_format	mt1.mtb.mtb_format
-#define mti_mutex	mt1.mtb.mtb_mutex
+#define mti_rmutex	mt1.mtb.mtb_rmutex
 #define mti_rmname	mt1.mtb.mtb_rmname
 #define mti_txnid	mt1.mtb.mtb_txnid
 #define mti_numreaders	mt1.mtb.mtb_numreaders
+#define mti_flags	mt1.mtb.mtb_flags
 		char pad[(sizeof(MDB_txbody)+CACHELINE-1) & ~(CACHELINE-1)];
 	} mt1;
 	union {
@@ -1137,11 +1156,11 @@ struct MDB_env {
 	int		me_live_reader;		/**< have liveness lock in reader table */
 #ifdef _WIN32
 	int		me_pidquery;		/**< Used in OpenProcess */
-	HANDLE		me_rmutex;		/* Windows mutexes don't reside in shared mem */
-	HANDLE		me_wmutex;
-#elif defined(MDB_USE_POSIX_SEM)
-	sem_t		*me_rmutex;		/* Shared mutexes are not supported */
-	sem_t		*me_wmutex;
+#endif
+#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM)
+	/* Windows mutexes/POSIX semaphores do not reside in shared mem */
+	mdb_mutex_t	me_rmutex;
+	mdb_mutex_t	me_wmutex;
 #endif
 	void		*me_userctx;	 /**< User-settable context */
 	MDB_assert_func *me_assert_func; /**< Callback for assertion failures */
@@ -1230,6 +1249,7 @@ static void	mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node);
 
 static int	mdb_drop0(MDB_cursor *mc, int subs);
 static void mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi);
+static int mdb_reader_check0(MDB_env *env, int rlocked, int *dead);
 
 /** @cond */
 static MDB_cmp_func	mdb_cmp_memn, mdb_cmp_memnr, mdb_cmp_int, mdb_cmp_cint, mdb_cmp_long;
@@ -1257,7 +1277,7 @@ static char *const mdb_errstr[] = {
 	"MDB_NOTFOUND: No matching key/data pair found",
 	"MDB_PAGE_NOTFOUND: Requested page not found",
 	"MDB_CORRUPTED: Located page was wrong type",
-	"MDB_PANIC: Update of meta page failed",
+	"MDB_PANIC: Update of meta page failed or environment had fatal error",
 	"MDB_VERSION_MISMATCH: Database environment version mismatch",
 	"MDB_INVALID: File is not an LMDB file",
 	"MDB_MAP_FULL: Environment mapsize limit reached",
@@ -2490,6 +2510,7 @@ mdb_txn_renew0(MDB_txn *txn)
 			} else {
 				MDB_PID_T pid = env->me_pid;
 				MDB_THR_T tid = pthread_self();
+				mdb_mutex_t rmutex = MDB_MUTEX(env, r);
 
 				if (!env->me_live_reader) {
 					rc = mdb_reader_pid(env, Pidset, pid);
@@ -2498,13 +2519,14 @@ mdb_txn_renew0(MDB_txn *txn)
 					env->me_live_reader = 1;
 				}
 
-				LOCK_MUTEX_R(env);
+				if (LOCK_MUTEX(rc, env, rmutex))
+					return rc;
 				nr = ti->mti_numreaders;
 				for (i=0; i<nr; i++)
 					if (ti->mti_readers[i].mr_pid == 0)
 						break;
 				if (i == env->me_maxreaders) {
-					UNLOCK_MUTEX_R(env);
+					UNLOCK_MUTEX(rmutex);
 					return MDB_READERS_FULL;
 				}
 				ti->mti_readers[i].mr_pid = pid;
@@ -2513,7 +2535,7 @@ mdb_txn_renew0(MDB_txn *txn)
 					ti->mti_numreaders = ++nr;
 				/* Save numreaders for un-mutexed mdb_env_close() */
 				env->me_numreaders = nr;
-				UNLOCK_MUTEX_R(env);
+				UNLOCK_MUTEX(rmutex);
 
 				r = &ti->mti_readers[i];
 				new_notls = (env->me_flags & MDB_NOTLS);
@@ -2528,7 +2550,9 @@ mdb_txn_renew0(MDB_txn *txn)
 		}
 	} else {
 		if (ti) {
-			LOCK_MUTEX_W(env);
+			mdb_mutex_t wmutex = MDB_MUTEX(env, w);
+			if (LOCK_MUTEX(rc, env, wmutex))
+				return rc;
 
 			txn->mt_txnid = ti->mti_txnid;
 			meta = env->me_metas[txn->mt_txnid & 1];
@@ -2798,7 +2822,7 @@ mdb_txn_reset0(MDB_txn *txn, const char *act)
 		env->me_txn = NULL;
 		/* The writer mutex was locked in mdb_txn_begin. */
 		if (env->me_txns)
-			UNLOCK_MUTEX_W(env);
+			UNLOCK_MUTEX(MDB_MUTEX(env, w));
 	}
 }
 
@@ -3382,7 +3406,7 @@ done:
 	mdb_dbis_update(txn, 1);
 
 	if (env->me_txns)
-		UNLOCK_MUTEX_W(env);
+		UNLOCK_MUTEX(MDB_MUTEX(env, w));
 	if (txn != env->me_txn0)
 		free(txn);
 
@@ -4293,6 +4317,7 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 		if (!env->me_rmutex) goto fail_errno;
 		env->me_wmutex = CreateMutex(&mdb_all_sa, FALSE, env->me_txns->mti_wmname);
 		if (!env->me_wmutex) goto fail_errno;
+		env->me_flags |= MDB_ROBUST;
 #elif defined(MDB_USE_POSIX_SEM)
 		struct stat stbuf;
 		struct {
@@ -4332,16 +4357,24 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 
 		if ((rc = pthread_mutexattr_init(&mattr))
 			|| (rc = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED))
-			|| (rc = pthread_mutex_init(&env->me_txns->mti_mutex, &mattr))
+#ifdef MDB_ROBUST_SUPPORTED
+			|| ((env->me_flags & MDB_ROBUST) &&
+				(rc = pthread_mutexattr_setrobust(&mattr, PTHREAD_MUTEX_ROBUST)))
+#endif
+			|| (rc = pthread_mutex_init(&env->me_txns->mti_rmutex, &mattr))
 			|| (rc = pthread_mutex_init(&env->me_txns->mti_wmutex, &mattr)))
 			goto fail;
 		pthread_mutexattr_destroy(&mattr);
 #endif	/* _WIN32 || MDB_USE_POSIX_SEM */
+#ifndef MDB_ROBUST_SUPPORTED
+		env->me_flags &= ~MDB_ROBUST;
+#endif
 
 		env->me_txns->mti_magic = MDB_MAGIC;
 		env->me_txns->mti_format = MDB_LOCK_FORMAT;
 		env->me_txns->mti_txnid = 0;
 		env->me_txns->mti_numreaders = 0;
+		env->me_txns->mti_flags = env->me_flags;
 
 	} else {
 		if (env->me_txns->mti_magic != MDB_MAGIC) {
@@ -4359,6 +4392,8 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 		if (rc && rc != EACCES && rc != EAGAIN) {
 			goto fail;
 		}
+		env->me_flags = (env->me_flags & ~MDB_ROBUST) |
+			(env->me_txns->mti_flags & MDB_ROBUST);
 #ifdef _WIN32
 		env->me_rmutex = OpenMutex(SYNCHRONIZE, FALSE, env->me_txns->mti_rmname);
 		if (!env->me_rmutex) goto fail_errno;
@@ -4390,8 +4425,13 @@ fail:
 	 *	environment and re-opening it with the new flags.
 	 */
 #define	CHANGEABLE	(MDB_NOSYNC|MDB_NOMETASYNC|MDB_MAPASYNC|MDB_NOMEMINIT)
-#define	CHANGELESS	(MDB_FIXEDMAP|MDB_NOSUBDIR|MDB_RDONLY|MDB_WRITEMAP| \
-	MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD)
+#define	CHANGELESS	(MDB_FIXEDMAP|MDB_NOSUBDIR|MDB_RDONLY|ROBUST_FLAG| \
+	MDB_WRITEMAP|MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD)
+#ifdef MDB_ROBUST_SUPPORTED
+#define ROBUST_FLAG MDB_ROBUST
+#else
+#define ROBUST_FLAG 0
+#endif
 
 #if VALID_FLAGS & PERSISTENT_FLAGS & (CHANGEABLE|CHANGELESS)
 # error "Persistent DB flags & env flags overlap, but both go in mm_flags"
@@ -4632,7 +4672,6 @@ mdb_env_close0(MDB_env *env, int excl)
 	env->me_flags &= ~(MDB_ENV_ACTIVE|MDB_ENV_TXKEY);
 }
 
-
 void ESECT
 mdb_env_close(MDB_env *env)
 {
@@ -8622,6 +8661,7 @@ static int ESECT
 mdb_env_copyfd0(MDB_env *env, HANDLE fd)
 {
 	MDB_txn *txn = NULL;
+	mdb_mutex_t wmutex = NULL;
 	int rc;
 	size_t wsize;
 	char *ptr;
@@ -8646,11 +8686,13 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd)
 		mdb_txn_reset0(txn, "reset-stage1");
 
 		/* Temporarily block writers until we snapshot the meta pages */
-		LOCK_MUTEX_W(env);
+		wmutex = MDB_MUTEX(env, w);
+		if (LOCK_MUTEX(rc, env, wmutex))
+			goto leave;
 
 		rc = mdb_txn_renew0(txn);
 		if (rc) {
-			UNLOCK_MUTEX_W(env);
+			UNLOCK_MUTEX(wmutex);
 			goto leave;
 		}
 	}
@@ -8674,8 +8716,8 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd)
 			break;
 		}
 	}
-	if (env->me_txns)
-		UNLOCK_MUTEX_W(env);
+	if (wmutex)
+		UNLOCK_MUTEX(wmutex);
 
 	if (rc)
 		goto leave;
@@ -8806,7 +8848,7 @@ mdb_env_copy(MDB_env *env, const char *path)
 int ESECT
 mdb_env_set_flags(MDB_env *env, unsigned int flag, int onoff)
 {
-	if ((flag & CHANGEABLE) != flag)
+	if (flag & (env->me_map ? ~CHANGEABLE : ~(CHANGEABLE|CHANGELESS)))
 		return EINVAL;
 	if (onoff)
 		env->me_flags |= flag;
@@ -9360,17 +9402,22 @@ mdb_pid_insert(MDB_PID_T *ids, MDB_PID_T pid)
 int ESECT
 mdb_reader_check(MDB_env *env, int *dead)
 {
-	unsigned int i, j, rdrs;
-	MDB_reader *mr;
-	MDB_PID_T *pids, pid;
-	int count = 0;
-
 	if (!env)
 		return EINVAL;
 	if (dead)
 		*dead = 0;
-	if (!env->me_txns)
-		return MDB_SUCCESS;
+	return env->me_txns ? mdb_reader_check0(env, 0, dead) : MDB_SUCCESS;
+}
+
+/** As #mdb_reader_check(). rlocked = <caller locked the reader mutex>. */
+static int mdb_reader_check0(MDB_env *env, int rlocked, int *dead)
+{
+	mdb_mutex_t rmutex = rlocked ? NULL : MDB_MUTEX(env, r);
+	unsigned int i, j, rdrs;
+	MDB_reader *mr;
+	pid_t *pids, pid;
+	int rc = MDB_SUCCESS, count = 0;
+
 	rdrs = env->me_txns->mti_numreaders;
 	pids = malloc((rdrs+1) * sizeof(MDB_PID_T));
 	if (!pids)
@@ -9378,22 +9425,32 @@ mdb_reader_check(MDB_env *env, int *dead)
 	pids[0] = 0;
 	mr = env->me_txns->mti_readers;
 	for (i=0; i<rdrs; i++) {
-		if (mr[i].mr_pid && mr[i].mr_pid != env->me_pid) {
-			pid = mr[i].mr_pid;
+		pid = mr[i].mr_pid;
+		if (pid && pid != env->me_pid) {
 			if (mdb_pid_insert(pids, pid) == 0) {
 				if (!mdb_reader_pid(env, Pidcheck, pid)) {
-					LOCK_MUTEX_R(env);
-					/* Recheck, a new process may have reused pid */
-					if (!mdb_reader_pid(env, Pidcheck, pid)) {
-						for (j=i; j<rdrs; j++)
+					/* Stale reader found */
+					j = i;
+					if (rmutex) {
+						if ((rc = LOCK_MUTEX0(rmutex)) != 0) {
+							if ((rc = mdb_mutex_failed(env, rmutex, rc)))
+								break;
+							rdrs = 0; /* the above checked all readers */
+						} else {
+							/* Recheck, a new process may have reused pid */
+							if (mdb_reader_pid(env, Pidcheck, pid))
+								j = rdrs;
+						}
+					}
+					for (; j<rdrs; j++)
 							if (mr[j].mr_pid == pid) {
 								DPRINTF(("clear stale reader pid %u txn %"Z"d",
 									(unsigned) pid, mr[j].mr_txnid));
 								mr[j].mr_pid = 0;
 								count++;
 							}
-					}
-					UNLOCK_MUTEX_R(env);
+					if (rmutex)
+						UNLOCK_MUTEX(rmutex);
 				}
 			}
 		}
@@ -9401,6 +9458,59 @@ mdb_reader_check(MDB_env *env, int *dead)
 	free(pids);
 	if (dead)
 		*dead = count;
-	return MDB_SUCCESS;
+	return rc;
 }
+
+#ifdef MDB_ROBUST_SUPPORTED
+/** Handle #LOCK_MUTEX0() failure.
+ * With #MDB_ROBUST, try to repair the lock file if the mutex owner died.
+ * @param[in] env	the environment handle
+ * @param[in] mutex	LOCK_MUTEX0() mutex
+ * @param[in] rc	LOCK_MUTEX0() error (nonzero)
+ * @return 0 on success with the mutex locked, or an error code on failure.
+ */
+static int mdb_mutex_failed(MDB_env *env, mdb_mutex_t mutex, int rc)
+{
+	int toggle, rlocked, rc2;
+#ifndef _WIN32
+	enum { WAIT_ABANDONED = EOWNERDEAD };
+#endif
+
+	if (rc == (int) WAIT_ABANDONED) {
+		/* We own the mutex. Clean up after dead previous owner. */
+		rc = MDB_SUCCESS;
+		rlocked = (mutex == MDB_MUTEX(env, r));
+		if (!rlocked) {
+			/* Keep mti_txnid updated, otherwise next writer can
+			 * overwrite data which latest meta page refers to.
+			 * TODO: Instead revert any aborted commit and sync?
+			 */
+			toggle = mdb_env_pick_meta(env);
+			env->me_txns->mti_txnid = env->me_metas[toggle]->mm_txnid;
+			/* env is hosed if the dead thread was ours */
+			if (env->me_txn) {
+				env->me_flags |= MDB_FATAL_ERROR;
+				env->me_txn = NULL;
+				rc = MDB_PANIC;
+			}
+		}
+		DPRINTF(("%cmutex owner died, %s", (rlocked ? 'r' : 'w'),
+			(rc ? "this process' env is hosed" : "recovering")));
+		rc2 = mdb_reader_check0(env, rlocked, NULL);
+		if (rc2 == 0)
+			rc2 = pthread_mutex_consistent(mutex);
+		if (rc || (rc = rc2)) {
+			DPRINTF(("LOCK_MUTEX recovery failed, %s", mdb_strerror(rc)));
+			UNLOCK_MUTEX(mutex);
+		}
+	} else {
+#ifdef _WIN32
+		rc = ErrCode();
+#endif
+		DPRINTF(("LOCK_MUTEX failed, %s", mdb_strerror(rc)));
+	}
+
+	return rc;
+}
+#endif	/* MDB_ROBUST_SUPPORTED */
 /** @} */

From f6add293a726aec3de809142e5fdaf836843eed9 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@symas.com>
Date: Wed, 16 Jul 2014 09:24:45 -0700
Subject: [PATCH 002/504] Free write mutex earlier in txn_reset0

---
 libraries/liblmdb/mdb.c | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 70c4de263a..b36f3b1281 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2798,12 +2798,26 @@ mdb_txn_reset0(MDB_txn *txn, const char *act)
 		txn->mt_numdbs = 0;		/* close nothing if called again */
 		txn->mt_dbxs = NULL;	/* mark txn as reset */
 	} else {
+		pgno_t *pghead = env->me_pghead;
+		env->me_pghead = NULL;
+		env->me_pglast = 0;
+
+		if (!txn->mt_parent) {
+			if (mdb_midl_shrink(&txn->mt_free_pgs))
+				env->me_free_pgs = txn->mt_free_pgs;
+
+			env->me_txn = NULL;
+			/* The writer mutex was locked in mdb_txn_begin. */
+			if (env->me_txns)
+				UNLOCK_MUTEX(MDB_MUTEX(env, w));
+		}
+
 		mdb_cursors_close(txn, 0);
 
 		if (!(env->me_flags & MDB_WRITEMAP)) {
 			mdb_dlist_free(txn);
 		}
-		mdb_midl_free(env->me_pghead);
+		mdb_midl_free(pghead);
 
 		if (txn->mt_parent) {
 			txn->mt_parent->mt_child = NULL;
@@ -2811,18 +2825,7 @@ mdb_txn_reset0(MDB_txn *txn, const char *act)
 			mdb_midl_free(txn->mt_free_pgs);
 			mdb_midl_free(txn->mt_spill_pgs);
 			free(txn->mt_u.dirty_list);
-			return;
 		}
-
-		if (mdb_midl_shrink(&txn->mt_free_pgs))
-			env->me_free_pgs = txn->mt_free_pgs;
-		env->me_pghead = NULL;
-		env->me_pglast = 0;
-
-		env->me_txn = NULL;
-		/* The writer mutex was locked in mdb_txn_begin. */
-		if (env->me_txns)
-			UNLOCK_MUTEX(MDB_MUTEX(env, w));
 	}
 }
 

From ec6cf4e40fb6565a3717e6eb2a4febe143e663d0 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <h.b.furuseth@usit.uio.no>
Date: Sun, 20 Jul 2014 16:42:04 +0200
Subject: [PATCH 003/504] MDB_ROBUST: Use MDB_PID_T. Drop a wmutex variable.

---
 libraries/liblmdb/mdb.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index b36f3b1281..cc15a1bfd1 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2550,8 +2550,7 @@ mdb_txn_renew0(MDB_txn *txn)
 		}
 	} else {
 		if (ti) {
-			mdb_mutex_t wmutex = MDB_MUTEX(env, w);
-			if (LOCK_MUTEX(rc, env, wmutex))
+			if (LOCK_MUTEX(rc, env, MDB_MUTEX(env, w)))
 				return rc;
 
 			txn->mt_txnid = ti->mti_txnid;
@@ -9418,7 +9417,7 @@ static int mdb_reader_check0(MDB_env *env, int rlocked, int *dead)
 	mdb_mutex_t rmutex = rlocked ? NULL : MDB_MUTEX(env, r);
 	unsigned int i, j, rdrs;
 	MDB_reader *mr;
-	pid_t *pids, pid;
+	MDB_PID_T *pids, pid;
 	int rc = MDB_SUCCESS, count = 0;
 
 	rdrs = env->me_txns->mti_numreaders;

From a53716ed968671a8df5e1fd6ca23cb959cf61e2d Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@symas.com>
Date: Sun, 20 Jul 2014 08:10:34 -0700
Subject: [PATCH 004/504] Must do dlist_free before releasing Wmutex

---
 libraries/liblmdb/mdb.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index cc15a1bfd1..6179b57868 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2801,6 +2801,10 @@ mdb_txn_reset0(MDB_txn *txn, const char *act)
 		env->me_pghead = NULL;
 		env->me_pglast = 0;
 
+		if (!(env->me_flags & MDB_WRITEMAP)) {
+			mdb_dlist_free(txn);
+		}
+
 		if (!txn->mt_parent) {
 			if (mdb_midl_shrink(&txn->mt_free_pgs))
 				env->me_free_pgs = txn->mt_free_pgs;
@@ -2813,9 +2817,6 @@ mdb_txn_reset0(MDB_txn *txn, const char *act)
 
 		mdb_cursors_close(txn, 0);
 
-		if (!(env->me_flags & MDB_WRITEMAP)) {
-			mdb_dlist_free(txn);
-		}
 		mdb_midl_free(pghead);
 
 		if (txn->mt_parent) {

From 58ddb5527bd4868bb7017cfe2051bc2e24bcf5a8 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@symas.com>
Date: Sun, 20 Jul 2014 11:08:55 -0700
Subject: [PATCH 005/504] Use SysV semaphores instead of POSIX

Since they can cleanup after themselves on process exit
---
 libraries/liblmdb/mdb.c      | 167 ++++++++++++++++++-----------------
 libraries/liblmdb/mdb_copy.c |   2 +-
 2 files changed, 88 insertions(+), 81 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 6179b57868..a7e6dd7055 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -102,7 +102,7 @@ extern int cacheflush(char *addr, int nbytes, int cache);
 #endif
 
 #if defined(__APPLE__) || defined (BSD)
-# define MDB_USE_POSIX_SEM	1
+# define MDB_USE_SYSV_SEM	1
 # define MDB_FDATASYNC		fsync
 #elif defined(ANDROID)
 # define MDB_FDATASYNC		fsync
@@ -110,11 +110,18 @@ extern int cacheflush(char *addr, int nbytes, int cache);
 
 #ifndef _WIN32
 #include <pthread.h>
-#ifdef MDB_USE_POSIX_SEM
-# define MDB_USE_HASH		1
-#include <semaphore.h>
-#endif
-#endif
+#ifdef MDB_USE_SYSV_SEM
+#include <sys/ipc.h>
+#include <sys/sem.h>
+#ifdef _SEM_SEMUN_UNDEFINED
+union semun {
+	int val;
+	struct semid_ds *buf;
+	unsigned short *array;
+};
+#endif /* _SEM_SEMUN_UNDEFINED */
+#endif /* MDB_USE_SYSV_SEM */
+#endif /* !_WIN32 */
 
 #ifdef USE_VALGRIND
 #include <valgrind/memcheck.h>
@@ -193,7 +200,7 @@ extern int cacheflush(char *addr, int nbytes, int cache);
 #define MDB_DEVEL 0
 #endif
 
-#if MDB_DEVEL && (defined(_WIN32) || (defined(EOWNERDEAD) && !defined(MDB_USE_POSIX_SEM)))
+#if MDB_DEVEL && (defined(_WIN32) || (defined(EOWNERDEAD) && !defined(MDB_USE_SYSV_SEM)))
 #define MDB_ROBUST_SUPPORTED	1
 #endif
 
@@ -254,25 +261,30 @@ typedef HANDLE mdb_mutex_t;
 	/** For MDB_LOCK_FORMAT: True if readers take a pid lock in the lockfile */
 #define MDB_PIDLOCK			1
 
-#ifdef MDB_USE_POSIX_SEM
+#ifdef MDB_USE_SYSV_SEM
 
-typedef sem_t *mdb_mutex_t;
-#define MDB_MUTEX(env, rw)		((env)->me_##rw##mutex)
+typedef struct mdb_mutex {
+	int semid;
+	int semnum;
+} mdb_mutex_t;
+
+#define MDB_MUTEX(env, rw)		(&(env)->me_##rw##mutex)
 #define LOCK_MUTEX0(mutex)		mdb_sem_wait(mutex)
-#define UNLOCK_MUTEX(mutex)		sem_post(mutex)
+#define UNLOCK_MUTEX(mutex)		do { struct sembuf sb = { mutex->semnum, 1, SEM_UNDO }; semop(mutex->semid, &sb, 1); } while(0)
 
 static int
-mdb_sem_wait(sem_t *sem)
+mdb_sem_wait(mdb_mutex_t *sem)
 {
    int rc;
-   while ((rc = sem_wait(sem)) && (rc = errno) == EINTR) ;
+   struct sembuf sb = { sem->semnum, -1, SEM_UNDO };
+   while ((rc = semop(sem->semid, &sb, 1)) && (rc = errno) == EINTR) ;
    return rc;
 }
 
 #else
 	/** Pointer/HANDLE type of shared mutex/semaphore.
 	 */
-typedef pthread_mutex_t *mdb_mutex_t;
+typedef pthread_mutex_t mdb_mutex_t;
 	/** Mutex for the reader table (rw = r) or write transaction (rw = w).
 	 */
 #define MDB_MUTEX(env, rw)	(&(env)->me_txns->mti_##rw##mutex)
@@ -283,7 +295,7 @@ typedef pthread_mutex_t *mdb_mutex_t;
 	/** Unlock the reader or writer mutex.
 	 */
 #define UNLOCK_MUTEX(mutex)	pthread_mutex_unlock(mutex)
-#endif	/* MDB_USE_POSIX_SEM */
+#endif	/* MDB_USE_SYSV_SEM */
 
 	/** Get the error code for the last failed system function.
 	 */
@@ -308,8 +320,10 @@ typedef pthread_mutex_t *mdb_mutex_t;
 #define	GET_PAGESIZE(x)	((x) = sysconf(_SC_PAGE_SIZE))
 #endif
 
-#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM)
+#if defined(_WIN32)
 #define MNAME_LEN	32
+#elif defined(MDB_USE_SYSV_SEM)
+#define MNAME_LEN	0
 #else
 #define MNAME_LEN	(sizeof(pthread_mutex_t))
 #endif
@@ -323,7 +337,7 @@ typedef pthread_mutex_t *mdb_mutex_t;
 #define LOCK_MUTEX(rc, env, mutex) \
 	(((rc) = LOCK_MUTEX0(mutex)) && \
 	 ((rc) = mdb_mutex_failed(env, mutex, rc)))
-static int mdb_mutex_failed(MDB_env *env, mdb_mutex_t mutex, int rc);
+static int mdb_mutex_failed(MDB_env *env, mdb_mutex_t *mutex, int rc);
 #else
 #define LOCK_MUTEX(rc, env, mutex) ((rc) = LOCK_MUTEX0(mutex))
 #define mdb_mutex_failed(env, mutex, rc) (rc)
@@ -636,8 +650,10 @@ typedef struct MDB_txbody {
 	uint32_t	mtb_magic;
 		/** Format of this lock file. Must be set to #MDB_LOCK_FORMAT. */
 	uint32_t	mtb_format;
-#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM)
+#if defined(_WIN32)
 	char	mtb_rmname[MNAME_LEN];
+#elif defined(MDB_USE_SYSV_SEM)
+	int 	mtb_semid;
 #else
 		/** Mutex protecting access to this table.
 		 *	This is the #MDB_MUTEX(env,r) reader table lock.
@@ -672,9 +688,11 @@ typedef struct MDB_txninfo {
 		char pad[(sizeof(MDB_txbody)+CACHELINE-1) & ~(CACHELINE-1)];
 	} mt1;
 	union {
-#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM)
+#if defined(_WIN32)
 		char mt2_wmname[MNAME_LEN];
 #define	mti_wmname	mt2.mt2_wmname
+#elif defined(MDB_USE_SYSV_SEM)
+#define	mti_semid	mt1.mtb.mtb_semid
 #else
 		pthread_mutex_t	mt2_wmutex;
 #define mti_wmutex	mt2.mt2_wmutex
@@ -1157,8 +1175,8 @@ struct MDB_env {
 #ifdef _WIN32
 	int		me_pidquery;		/**< Used in OpenProcess */
 #endif
-#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM)
-	/* Windows mutexes/POSIX semaphores do not reside in shared mem */
+#if defined(_WIN32) || defined(MDB_USE_SYSV_SEM)
+	/* Windows mutexes/SysV semaphores do not reside in shared mem */
 	mdb_mutex_t	me_rmutex;
 	mdb_mutex_t	me_wmutex;
 #endif
@@ -1212,7 +1230,7 @@ static int	mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata,
 static int  mdb_env_read_header(MDB_env *env, MDB_meta *meta);
 static int  mdb_env_pick_meta(const MDB_env *env);
 static int  mdb_env_write_meta(MDB_txn *txn);
-#if !(defined(_WIN32) || defined(MDB_USE_POSIX_SEM)) /* Drop unused excl arg */
+#if !(defined(_WIN32) || defined(MDB_USE_SYSV_SEM)) /* Drop unused excl arg */
 # define mdb_env_close0(env, excl) mdb_env_close1(env)
 #endif
 static void mdb_env_close0(MDB_env *env, int excl);
@@ -2510,7 +2528,7 @@ mdb_txn_renew0(MDB_txn *txn)
 			} else {
 				MDB_PID_T pid = env->me_pid;
 				MDB_THR_T tid = pthread_self();
-				mdb_mutex_t rmutex = MDB_MUTEX(env, r);
+				mdb_mutex_t *rmutex = MDB_MUTEX(env, r);
 
 				if (!env->me_live_reader) {
 					rc = mdb_reader_pid(env, Pidset, pid);
@@ -3695,9 +3713,9 @@ mdb_env_create(MDB_env **env)
 	e->me_fd = INVALID_HANDLE_VALUE;
 	e->me_lfd = INVALID_HANDLE_VALUE;
 	e->me_mfd = INVALID_HANDLE_VALUE;
-#ifdef MDB_USE_POSIX_SEM
-	e->me_rmutex = SEM_FAILED;
-	e->me_wmutex = SEM_FAILED;
+#ifdef MDB_USE_SYSV_SEM
+	e->me_rmutex.semid = -1;
+	e->me_wmutex.semid = -1;
 #endif
 	e->me_pid = getpid();
 	GET_PAGESIZE(e->me_os_psize);
@@ -4080,8 +4098,8 @@ mdb_env_excl_lock(MDB_env *env, int *excl)
 	if (!rc) {
 		*excl = 1;
 	} else
-# ifdef MDB_USE_POSIX_SEM
-	if (*excl < 0) /* always true when !MDB_USE_POSIX_SEM */
+# ifdef MDB_USE_SYSV_SEM
+	if (*excl < 0) /* always true when !MDB_USE_SYSV_SEM */
 # endif
 	{
 		lock_info.l_type = F_RDLCK;
@@ -4321,41 +4339,23 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 		env->me_wmutex = CreateMutex(&mdb_all_sa, FALSE, env->me_txns->mti_wmname);
 		if (!env->me_wmutex) goto fail_errno;
 		env->me_flags |= MDB_ROBUST;
-#elif defined(MDB_USE_POSIX_SEM)
-		struct stat stbuf;
-		struct {
-			dev_t dev;
-			ino_t ino;
-		} idbuf;
-		MDB_val val;
-		char encbuf[11];
+#elif defined(MDB_USE_SYSV_SEM)
+		union semun semu;
+		unsigned short vals[2] = {1, 1};
+		int semid = semget(IPC_PRIVATE, 2, mode);
+		if (semid < 0)
+			goto fail_errno;
 
-#if defined(__NetBSD__)
-#define	MDB_SHORT_SEMNAMES	1	/* limited to 14 chars */
-#endif
-		if (fstat(env->me_lfd, &stbuf)) goto fail_errno;
-		idbuf.dev = stbuf.st_dev;
-		idbuf.ino = stbuf.st_ino;
-		val.mv_data = &idbuf;
-		val.mv_size = sizeof(idbuf);
-		mdb_hash_enc(&val, encbuf);
-#ifdef MDB_SHORT_SEMNAMES
-		encbuf[9] = '\0';	/* drop name from 15 chars to 14 chars */
-#endif
-		sprintf(env->me_txns->mti_rmname, "/MDBr%s", encbuf);
-		sprintf(env->me_txns->mti_wmname, "/MDBw%s", encbuf);
-		/* Clean up after a previous run, if needed:  Try to
-		 * remove both semaphores before doing anything else.
-		 */
-		sem_unlink(env->me_txns->mti_rmname);
-		sem_unlink(env->me_txns->mti_wmname);
-		env->me_rmutex = sem_open(env->me_txns->mti_rmname,
-			O_CREAT|O_EXCL, mode, 1);
-		if (env->me_rmutex == SEM_FAILED) goto fail_errno;
-		env->me_wmutex = sem_open(env->me_txns->mti_wmname,
-			O_CREAT|O_EXCL, mode, 1);
-		if (env->me_wmutex == SEM_FAILED) goto fail_errno;
-#else	/* MDB_USE_POSIX_SEM */
+		env->me_rmutex.semid = semid;
+		env->me_wmutex.semid = semid;
+		env->me_rmutex.semnum = 0;
+		env->me_wmutex.semnum = 1;
+
+		semu.array = vals;
+		if (semctl(semid, 0, SETALL, semu) < 0)
+			goto fail_errno;
+		env->me_txns->mti_semid = semid;
+#else	/* MDB_USE_SYSV_SEM */
 		pthread_mutexattr_t mattr;
 
 		if ((rc = pthread_mutexattr_init(&mattr))
@@ -4368,7 +4368,7 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 			|| (rc = pthread_mutex_init(&env->me_txns->mti_wmutex, &mattr)))
 			goto fail;
 		pthread_mutexattr_destroy(&mattr);
-#endif	/* _WIN32 || MDB_USE_POSIX_SEM */
+#endif	/* _WIN32 || MDB_USE_SYSV_SEM */
 #ifndef MDB_ROBUST_SUPPORTED
 		env->me_flags &= ~MDB_ROBUST;
 #endif
@@ -4402,11 +4402,23 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 		if (!env->me_rmutex) goto fail_errno;
 		env->me_wmutex = OpenMutex(SYNCHRONIZE, FALSE, env->me_txns->mti_wmname);
 		if (!env->me_wmutex) goto fail_errno;
-#elif defined(MDB_USE_POSIX_SEM)
-		env->me_rmutex = sem_open(env->me_txns->mti_rmname, 0);
-		if (env->me_rmutex == SEM_FAILED) goto fail_errno;
-		env->me_wmutex = sem_open(env->me_txns->mti_wmname, 0);
-		if (env->me_wmutex == SEM_FAILED) goto fail_errno;
+#elif defined(MDB_USE_SYSV_SEM)
+		struct semid_ds buf;
+		union semun semu;
+		int semid = env->me_txns->mti_semid;
+		semu.buf = &buf;
+
+		/* check for read access */
+		if (semctl(semid, 0, IPC_STAT, semu) < 0)
+			goto fail_errno;
+		/* check for write access */
+		if (semctl(semid, 0, IPC_SET, semu) < 0)
+			goto fail_errno;
+
+		env->me_rmutex.semid = semid;
+		env->me_wmutex.semid = semid;
+		env->me_rmutex.semnum = 0;
+		env->me_wmutex.semnum = 1;
 #endif
 	}
 	return MDB_SUCCESS;
@@ -4642,20 +4654,15 @@ mdb_env_close0(MDB_env *env, int excl)
 		/* Windows automatically destroys the mutexes when
 		 * the last handle closes.
 		 */
-#elif defined(MDB_USE_POSIX_SEM)
-		if (env->me_rmutex != SEM_FAILED) {
-			sem_close(env->me_rmutex);
-			if (env->me_wmutex != SEM_FAILED)
-				sem_close(env->me_wmutex);
+#elif defined(MDB_USE_SYSV_SEM)
+		if (env->me_rmutex.semid != -1) {
 			/* If we have the filelock:  If we are the
 			 * only remaining user, clean up semaphores.
 			 */
 			if (excl == 0)
 				mdb_env_excl_lock(env, &excl);
-			if (excl > 0) {
-				sem_unlink(env->me_txns->mti_rmname);
-				sem_unlink(env->me_txns->mti_wmname);
-			}
+			if (excl > 0)
+				semctl(env->me_rmutex.semid, 0, IPC_RMID);
 		}
 #endif
 		munmap((void *)env->me_txns, (env->me_maxreaders-1)*sizeof(MDB_reader)+sizeof(MDB_txninfo));
@@ -8664,7 +8671,7 @@ static int ESECT
 mdb_env_copyfd0(MDB_env *env, HANDLE fd)
 {
 	MDB_txn *txn = NULL;
-	mdb_mutex_t wmutex = NULL;
+	mdb_mutex_t *wmutex = NULL;
 	int rc;
 	size_t wsize;
 	char *ptr;
@@ -9415,7 +9422,7 @@ mdb_reader_check(MDB_env *env, int *dead)
 /** As #mdb_reader_check(). rlocked = <caller locked the reader mutex>. */
 static int mdb_reader_check0(MDB_env *env, int rlocked, int *dead)
 {
-	mdb_mutex_t rmutex = rlocked ? NULL : MDB_MUTEX(env, r);
+	mdb_mutex_t *rmutex = rlocked ? NULL : MDB_MUTEX(env, r);
 	unsigned int i, j, rdrs;
 	MDB_reader *mr;
 	MDB_PID_T *pids, pid;
@@ -9472,7 +9479,7 @@ static int mdb_reader_check0(MDB_env *env, int rlocked, int *dead)
  * @param[in] rc	LOCK_MUTEX0() error (nonzero)
  * @return 0 on success with the mutex locked, or an error code on failure.
  */
-static int mdb_mutex_failed(MDB_env *env, mdb_mutex_t mutex, int rc)
+static int mdb_mutex_failed(MDB_env *env, mdb_mutex_t *mutex, int rc)
 {
 	int toggle, rlocked, rc2;
 #ifndef _WIN32
diff --git a/libraries/liblmdb/mdb_copy.c b/libraries/liblmdb/mdb_copy.c
index e7f965c03a..af0a941110 100644
--- a/libraries/liblmdb/mdb_copy.c
+++ b/libraries/liblmdb/mdb_copy.c
@@ -64,7 +64,7 @@ int main(int argc,char * argv[])
 	act = "opening environment";
 	rc = mdb_env_create(&env);
 	if (rc == MDB_SUCCESS) {
-		rc = mdb_env_open(env, argv[1], flags, 0664);
+		rc = mdb_env_open(env, argv[1], flags, 0600);
 	}
 	if (rc == MDB_SUCCESS) {
 		act = "copying";

From e3b6c359a935283c21e1fd7d03b790d87d53b933 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@symas.com>
Date: Wed, 1 Oct 2014 20:58:15 +0100
Subject: [PATCH 006/504] Use robust mutexes by default

Making it optional on pthreads just complicates things;
they're always robust on other implementations
---
 libraries/liblmdb/lmdb.h | 20 +++++++-------------
 libraries/liblmdb/mdb.c  | 30 ++++--------------------------
 2 files changed, 11 insertions(+), 39 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 82a5410ab1..4236218bb5 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -50,12 +50,14 @@
  *
  *	  Fix: Check for stale readers periodically, using the
  *	  #mdb_reader_check function or the \ref mdb_stat_1 "mdb_stat" tool.
- *	  Catch stale
- *	  locks with option MDB_ROBUST if supported (non-BSD). Or just
- *	  make all programs using the database close it; the lockfile
- *	  is always reset on first open of the environment.
+ *	  Stale writers will be cleared automatically on most systems:
+ *	  - Windows - automatic
+ *	  - BSD, systems using SysV semaphores - automatic
+ *	  - Linux, systems using POSIX mutexes with Robust option - automatic
+ *	  Otherwise just make all programs using the database close it;
+ *	  the lockfile is always reset on first open of the environment.
  *
- *	- On BSD systems or others configured with MDB_USE_POSIX_SEM,
+ *	- On BSD systems or others configured with MDB_USE_SYSV_SEM,
  *	  startup can fail due to semaphores owned by another userid.
  *
  *	  Fix: Open and close the database as the user which owns the
@@ -290,8 +292,6 @@ typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *rel
 #define MDB_NORDAHEAD	0x800000
 	/** don't initialize malloc'd memory before writing to datafile */
 #define MDB_NOMEMINIT	0x1000000
-	/** catch stale locks if supported (not on BSD, needs robust mutexes) */
-#define MDB_ROBUST		0x2000000
 /** @} */
 
 /**	@defgroup	mdb_dbi_open	Database Flags
@@ -516,12 +516,6 @@ int  mdb_env_create(MDB_env **env);
 	 *		Open the environment in read-only mode. No write operations will be
 	 *		allowed. LMDB will still modify the lock file - except on read-only
 	 *		filesystems, where LMDB does not use locks.
-	 *	<li>#MDB_ROBUST
-	 *		Initialize the lockfile to catch stale locks if robust mutexes
-	 *		are supported, so aborted processes will not block others.
-	 *		Ignored when another process has the environment open. Unsupported
-	 *		by liblmdb built with MDB_USE_POSIX_SEM (such as BSD systems).
-	 *		Enabled by default on Windows. Some locking slowdown on Unix.
 	 *	<li>#MDB_WRITEMAP
 	 *		Use a writeable memory map unless MDB_RDONLY is set. This is faster
 	 *		and uses fewer mallocs, but loses protection from application bugs
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index a7e6dd7055..11cdc33d12 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -200,7 +200,7 @@ union semun {
 #define MDB_DEVEL 0
 #endif
 
-#if MDB_DEVEL && (defined(_WIN32) || (defined(EOWNERDEAD) && !defined(MDB_USE_SYSV_SEM)))
+#if defined(WIN32) || (defined(EOWNERDEAD) && !defined(MDB_USE_SYSV_SEM))
 #define MDB_ROBUST_SUPPORTED	1
 #endif
 
@@ -670,8 +670,6 @@ typedef struct MDB_txbody {
 		 *	when readers release their slots.
 		 */
 	unsigned	mtb_numreaders;
-		/** Flags which the lock file was initialized with. */
-	unsigned	mtb_flags;
 } MDB_txbody;
 
 	/** The actual reader table definition. */
@@ -684,7 +682,6 @@ typedef struct MDB_txninfo {
 #define mti_rmname	mt1.mtb.mtb_rmname
 #define mti_txnid	mt1.mtb.mtb_txnid
 #define mti_numreaders	mt1.mtb.mtb_numreaders
-#define mti_flags	mt1.mtb.mtb_flags
 		char pad[(sizeof(MDB_txbody)+CACHELINE-1) & ~(CACHELINE-1)];
 	} mt1;
 	union {
@@ -4338,7 +4335,6 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 		if (!env->me_rmutex) goto fail_errno;
 		env->me_wmutex = CreateMutex(&mdb_all_sa, FALSE, env->me_txns->mti_wmname);
 		if (!env->me_wmutex) goto fail_errno;
-		env->me_flags |= MDB_ROBUST;
 #elif defined(MDB_USE_SYSV_SEM)
 		union semun semu;
 		unsigned short vals[2] = {1, 1};
@@ -4361,23 +4357,18 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 		if ((rc = pthread_mutexattr_init(&mattr))
 			|| (rc = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED))
 #ifdef MDB_ROBUST_SUPPORTED
-			|| ((env->me_flags & MDB_ROBUST) &&
-				(rc = pthread_mutexattr_setrobust(&mattr, PTHREAD_MUTEX_ROBUST)))
+			|| (rc = pthread_mutexattr_setrobust(&mattr, PTHREAD_MUTEX_ROBUST))
 #endif
 			|| (rc = pthread_mutex_init(&env->me_txns->mti_rmutex, &mattr))
 			|| (rc = pthread_mutex_init(&env->me_txns->mti_wmutex, &mattr)))
 			goto fail;
 		pthread_mutexattr_destroy(&mattr);
 #endif	/* _WIN32 || MDB_USE_SYSV_SEM */
-#ifndef MDB_ROBUST_SUPPORTED
-		env->me_flags &= ~MDB_ROBUST;
-#endif
 
 		env->me_txns->mti_magic = MDB_MAGIC;
 		env->me_txns->mti_format = MDB_LOCK_FORMAT;
 		env->me_txns->mti_txnid = 0;
 		env->me_txns->mti_numreaders = 0;
-		env->me_txns->mti_flags = env->me_flags;
 
 	} else {
 		if (env->me_txns->mti_magic != MDB_MAGIC) {
@@ -4395,8 +4386,6 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 		if (rc && rc != EACCES && rc != EAGAIN) {
 			goto fail;
 		}
-		env->me_flags = (env->me_flags & ~MDB_ROBUST) |
-			(env->me_txns->mti_flags & MDB_ROBUST);
 #ifdef _WIN32
 		env->me_rmutex = OpenMutex(SYNCHRONIZE, FALSE, env->me_txns->mti_rmname);
 		if (!env->me_rmutex) goto fail_errno;
@@ -4440,13 +4429,8 @@ fail:
 	 *	environment and re-opening it with the new flags.
 	 */
 #define	CHANGEABLE	(MDB_NOSYNC|MDB_NOMETASYNC|MDB_MAPASYNC|MDB_NOMEMINIT)
-#define	CHANGELESS	(MDB_FIXEDMAP|MDB_NOSUBDIR|MDB_RDONLY|ROBUST_FLAG| \
+#define	CHANGELESS	(MDB_FIXEDMAP|MDB_NOSUBDIR|MDB_RDONLY| \
 	MDB_WRITEMAP|MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD)
-#ifdef MDB_ROBUST_SUPPORTED
-#define ROBUST_FLAG MDB_ROBUST
-#else
-#define ROBUST_FLAG 0
-#endif
 
 #if VALID_FLAGS & PERSISTENT_FLAGS & (CHANGEABLE|CHANGELESS)
 # error "Persistent DB flags & env flags overlap, but both go in mm_flags"
@@ -9481,7 +9465,7 @@ static int mdb_reader_check0(MDB_env *env, int rlocked, int *dead)
  */
 static int mdb_mutex_failed(MDB_env *env, mdb_mutex_t *mutex, int rc)
 {
-	int toggle, rlocked, rc2;
+	int rlocked, rc2;
 #ifndef _WIN32
 	enum { WAIT_ABANDONED = EOWNERDEAD };
 #endif
@@ -9491,12 +9475,6 @@ static int mdb_mutex_failed(MDB_env *env, mdb_mutex_t *mutex, int rc)
 		rc = MDB_SUCCESS;
 		rlocked = (mutex == MDB_MUTEX(env, r));
 		if (!rlocked) {
-			/* Keep mti_txnid updated, otherwise next writer can
-			 * overwrite data which latest meta page refers to.
-			 * TODO: Instead revert any aborted commit and sync?
-			 */
-			toggle = mdb_env_pick_meta(env);
-			env->me_txns->mti_txnid = env->me_metas[toggle]->mm_txnid;
 			/* env is hosed if the dead thread was ours */
 			if (env->me_txn) {
 				env->me_flags |= MDB_FATAL_ERROR;

From 8bec53fdfd56528a26ec31f6361160585c91cb53 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@symas.com>
Date: Wed, 1 Oct 2014 21:00:03 +0100
Subject: [PATCH 007/504] Don't use -fPIC for static lib

---
 libraries/liblmdb/Makefile | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile
index 25c1095466..97b571747d 100644
--- a/libraries/liblmdb/Makefile
+++ b/libraries/liblmdb/Makefile
@@ -50,9 +50,9 @@ test:	all
 liblmdb.a:	mdb.o midl.o
 	ar rs $@ mdb.o midl.o
 
-liblmdb.so:	mdb.o midl.o
+liblmdb.so:	mdb.lo midl.lo
 #	$(CC) $(LDFLAGS) -pthread -shared -Wl,-Bsymbolic -o $@ mdb.o midl.o $(SOLIBS)
-	$(CC) $(LDFLAGS) -pthread -shared -o $@ mdb.o midl.o $(SOLIBS)
+	$(CC) $(LDFLAGS) -pthread -shared -o $@ mdb.lo midl.lo $(SOLIBS)
 
 mdb_stat: mdb_stat.o liblmdb.a
 mdb_copy: mdb_copy.o liblmdb.a
@@ -66,10 +66,16 @@ mtest5:	mtest5.o liblmdb.a
 mtest6:	mtest6.o liblmdb.a
 
 mdb.o: mdb.c lmdb.h midl.h
-	$(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -c mdb.c
+	$(CC) $(CFLAGS) $(CPPFLAGS) -c mdb.c
 
 midl.o: midl.c midl.h
-	$(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -c midl.c
+	$(CC) $(CFLAGS) $(CPPFLAGS) -c midl.c
+
+mdb.lo: mdb.c lmdb.h midl.h
+	$(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -c mdb.c -o $@
+
+midl.lo: midl.c midl.h
+	$(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -c midl.c -o $@
 
 %:	%.o
 	$(CC) $(CFLAGS) $(LDFLAGS) $^ $(LDLIBS) -o $@

From fd0315d6e58d42d945fbc1939faa5e2354bbe9a8 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@symas.com>
Date: Thu, 2 Oct 2014 21:33:35 +0100
Subject: [PATCH 008/504] ITS#7952 .gitignore mdb_dump/load

---
 libraries/liblmdb/.gitignore | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libraries/liblmdb/.gitignore b/libraries/liblmdb/.gitignore
index f3277afe42..bfa255a98c 100644
--- a/libraries/liblmdb/.gitignore
+++ b/libraries/liblmdb/.gitignore
@@ -3,6 +3,8 @@ mtest[23456]
 testdb
 mdb_copy
 mdb_stat
+mdb_dump
+mdb_load
 *.[ao]
 *.so
 *.exe

From 61dd017808f66ada27e89c58bc9567d5b2ade884 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@symas.com>
Date: Thu, 2 Oct 2014 21:35:15 +0100
Subject: [PATCH 009/504] ITS#7953 .gitignore *.lo

---
 libraries/liblmdb/.gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libraries/liblmdb/.gitignore b/libraries/liblmdb/.gitignore
index bfa255a98c..d5102a87c0 100644
--- a/libraries/liblmdb/.gitignore
+++ b/libraries/liblmdb/.gitignore
@@ -5,6 +5,7 @@ mdb_copy
 mdb_stat
 mdb_dump
 mdb_load
+*.lo
 *.[ao]
 *.so
 *.exe

From bfe297da47beb31debee48f016222b63d94271fd Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@symas.com>
Date: Thu, 2 Oct 2014 21:58:20 +0100
Subject: [PATCH 010/504] ITS#7955, #7671 fix MDB_PREV_DUP

---
 libraries/liblmdb/mdb.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 11cdc33d12..6a9fd97a5a 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5457,11 +5457,11 @@ mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
 					}
 					return rc;
 				}
-			} else {
-				mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF);
-				if (op == MDB_PREV_DUP)
-					return MDB_NOTFOUND;
 			}
+		} else {
+			mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF);
+			if (op == MDB_PREV_DUP)
+				return MDB_NOTFOUND;
 		}
 	}
 

From 56c2c160be19c555e4c42e459c8608ffaae7b150 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@symas.com>
Date: Thu, 2 Oct 2014 22:11:43 +0100
Subject: [PATCH 011/504] ITS#7956 fix compact of empty env

---
 libraries/liblmdb/mdb.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 6a9fd97a5a..3a86b57f9c 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -8618,8 +8618,12 @@ mdb_env_copyfd1(MDB_env *env, HANDLE fd)
 		/* Set metapage 1 */
 		mm->mm_last_pg = txn->mt_next_pgno - freecount - 1;
 		mm->mm_dbs[1] = txn->mt_dbs[1];
-		mm->mm_dbs[1].md_root = mm->mm_last_pg;
-		mm->mm_txnid = 1;
+		if (mm->mm_last_pg > 1) {
+			mm->mm_dbs[1].md_root = mm->mm_last_pg;
+			mm->mm_txnid = 1;
+		} else {
+			mm->mm_dbs[1].md_root = P_INVALID;
+		}
 	}
 	my.mc_wlen[0] = env->me_psize * 2;
 	my.mc_txn = txn;

From 461d9c8f34db8a1431257893916dbb7a6711a5a1 Mon Sep 17 00:00:00 2001
From: Leo Yuriev <leo@yuriev.ru>
Date: Fri, 5 Sep 2014 00:19:16 +0400
Subject: [PATCH 012/504] ITS#7841 trivial cleanup

TRIVIA - lmdb: clean testdb-dir while "make test".
---
 libraries/liblmdb/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile
index 97b571747d..5f8ef64461 100644
--- a/libraries/liblmdb/Makefile
+++ b/libraries/liblmdb/Makefile
@@ -44,7 +44,7 @@ clean:
 	rm -rf $(PROGS) *.[ao] *.so *~ testdb
 
 test:	all
-	mkdir testdb
+	rm -rf testdb && mkdir testdb
 	./mtest && ./mdb_stat testdb
 
 liblmdb.a:	mdb.o midl.o

From d77e3ddb4d210fc01586f4c776286c4b0f7b6611 Mon Sep 17 00:00:00 2001
From: Jean-Christophe DUBOIS <jcd@tribudubois.net>
Date: Fri, 3 Oct 2014 20:24:25 +0200
Subject: [PATCH 013/504] ITS#7959 Check fstat return value

Signed-off-by: Jean-Christophe DUBOIS <jcd@tribudubois.net>
---
 libraries/liblmdb/mdb.c | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 3a86b57f9c..5361e054f7 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -8724,14 +8724,23 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd)
 #ifdef WIN32
 	{
 		LARGE_INTEGER fsize;
-		GetFileSizeEx(env->me_fd, &fsize);
+
+		if (!GetFileSizeEx(env->me_fd, &fsize)) {
+			rc = ErrCode();
+			goto leave;
+		}
+
 		if (w2 > fsize.QuadPart)
 			w2 = fsize.QuadPart;
 	}
 #else
 	{
 		struct stat st;
-		fstat(env->me_fd, &st);
+
+		if ((rc = fstat(env->me_fd, &st))) {
+			goto leave;
+		}
+
 		if (w2 > (size_t)st.st_size)
 			w2 = st.st_size;
 	}

From d72b2f5d15319cb1424476dd3afc8528630461e4 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@symas.com>
Date: Sat, 4 Oct 2014 21:48:49 +0100
Subject: [PATCH 014/504] ITS#7961 fix txn init

More fallout from 4d02c741b120786df1b87ee9ed49c1d3f9bc7522
---
 libraries/liblmdb/mdb.c | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 5361e054f7..9728cb6af0 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2507,11 +2507,10 @@ mdb_txn_renew0(MDB_txn *txn)
 	uint16_t x;
 	int rc, new_notls = 0;
 
-	/* Setup db info */
-	txn->mt_numdbs = env->me_numdbs;
-	txn->mt_dbxs = env->me_dbxs;	/* mostly static anyway */
-
 	if (txn->mt_flags & MDB_TXN_RDONLY) {
+		/* Setup db info */
+		txn->mt_numdbs = env->me_numdbs;
+		txn->mt_dbxs = env->me_dbxs;	/* mostly static anyway */
 		if (!ti) {
 			meta = env->me_metas[ mdb_env_pick_meta(env) ];
 			txn->mt_txnid = meta->mm_txnid;
@@ -2574,6 +2573,8 @@ mdb_txn_renew0(MDB_txn *txn)
 			meta = env->me_metas[ mdb_env_pick_meta(env) ];
 			txn->mt_txnid = meta->mm_txnid;
 		}
+		/* Setup db info */
+		txn->mt_numdbs = env->me_numdbs;
 		txn->mt_txnid++;
 #if MDB_DEBUG
 		if (txn->mt_txnid == mdb_debug_start)
@@ -2660,18 +2661,17 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 		}
 		tsize = sizeof(MDB_ntxn);
 	}
-	size = tsize + env->me_maxdbs * (sizeof(MDB_db)+1);
+	size = tsize;
 	if (!(flags & MDB_RDONLY)) {
 		if (!parent) {
-			txn = env->me_txn0;
+			txn = env->me_txn0;	/* just reuse preallocated write txn */
 			txn->mt_flags = 0;
 			goto ok;
 		}
+		/* child txns use own copy of cursors */
 		size += env->me_maxdbs * sizeof(MDB_cursor *);
-		/* child txns use parent's dbiseqs */
-		if (!parent)
-			size += env->me_maxdbs * sizeof(unsigned int);
 	}
+	size += env->me_maxdbs * (sizeof(MDB_db)+1);
 
 	if ((txn = calloc(1, size)) == NULL) {
 		DPRINTF(("calloc: %s", strerror(errno)));
@@ -4565,6 +4565,7 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 				txn->mt_dbiseqs = (unsigned int *)(txn->mt_cursors + env->me_maxdbs);
 				txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs);
 				txn->mt_env = env;
+				txn->mt_dbxs = env->me_dbxs;
 				env->me_txn0 = txn;
 			} else {
 				rc = ENOMEM;

From 925e7805a5b2d4bc2d464ea5acd5ff38ff209bde Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@symas.com>
Date: Wed, 8 Oct 2014 19:25:27 +0100
Subject: [PATCH 015/504] ITS#7959 fix prev commit

fstat returns -1, not an errno. Move code into its own function
for reuse.
---
 libraries/liblmdb/mdb.c | 46 ++++++++++++++++++++++-------------------
 1 file changed, 25 insertions(+), 21 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 9728cb6af0..9e65c2741d 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -3866,6 +3866,27 @@ mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers)
 	return MDB_SUCCESS;
 }
 
+static int ESECT
+mdb_fsize(HANDLE fd, size_t *size)
+{
+#ifdef WIN32
+	LARGE_INTEGER fsize;
+
+	if (!GetFileSizeEx(fd, &fsize))
+		return ErrCode();
+
+	*size = fsize.QuadPart;
+#else
+	struct stat st;
+
+	if (fstat(fd, &st))
+		return ErrCode();
+
+	*size = st.st_size;
+#endif
+	return MDB_SUCCESS;
+}
+
 /** Further setup required for opening an LMDB environment
  */
 static int ESECT
@@ -8722,30 +8743,13 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd)
 		goto leave;
 
 	w2 = txn->mt_next_pgno * env->me_psize;
-#ifdef WIN32
 	{
-		LARGE_INTEGER fsize;
-
-		if (!GetFileSizeEx(env->me_fd, &fsize)) {
-			rc = ErrCode();
+		size_t fsize = 0;
+		if ((rc = mdb_fsize(env->me_fd, &fsize)))
 			goto leave;
-		}
-
-		if (w2 > fsize.QuadPart)
-			w2 = fsize.QuadPart;
+		if (w2 > fsize)
+			w2 = fsize;
 	}
-#else
-	{
-		struct stat st;
-
-		if ((rc = fstat(env->me_fd, &st))) {
-			goto leave;
-		}
-
-		if (w2 > (size_t)st.st_size)
-			w2 = st.st_size;
-	}
-#endif
 	wsize = w2 - wsize;
 	while (wsize > 0) {
 		if (wsize > MAX_WRITE)

From 9a80a8a8e8feed56fbccd8851b8a789f7fff9c11 Mon Sep 17 00:00:00 2001
From: "leo@yuriev.ru" <leo@yuriev.ru>
Date: Fri, 17 Oct 2014 22:22:39 +0000
Subject: [PATCH 016/504] ITS#7969 LMDB: volatile & __synchronize().

Globally shared fields of meta-data are not 'volatile'.
---
 libraries/liblmdb/mdb.c | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 9e65c2741d..c84c07f042 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -610,11 +610,11 @@ typedef struct MDB_rxbody {
 	 *	started from so we can avoid overwriting any data used in that
 	 *	particular version.
 	 */
-	txnid_t		mrb_txnid;
+	volatile txnid_t		mrb_txnid;
 	/** The process ID of the process owning this reader txn. */
-	MDB_PID_T	mrb_pid;
+	volatile MDB_PID_T	mrb_pid;
 	/** The thread ID of the thread owning this txn. */
-	MDB_THR_T	mrb_tid;
+	volatile MDB_THR_T	mrb_tid;
 } MDB_rxbody;
 
 	/** The actual reader record, with cacheline padding. */
@@ -664,12 +664,12 @@ typedef struct MDB_txbody {
 		 *	This is recorded here only for convenience; the value can always
 		 *	be determined by reading the main database meta pages.
 		 */
-	txnid_t		mtb_txnid;
+	volatile txnid_t		mtb_txnid;
 		/** The number of slots that have been used in the reader table.
 		 *	This always records the maximum count, it is not decremented
 		 *	when readers release their slots.
 		 */
-	unsigned	mtb_numreaders;
+	volatile unsigned	mtb_numreaders;
 } MDB_txbody;
 
 	/** The actual reader table definition. */
@@ -942,7 +942,7 @@ typedef struct MDB_meta {
 	/** Any persistent environment flags. @ref mdb_env */
 #define	mm_flags	mm_dbs[0].md_flags
 	pgno_t		mm_last_pg;			/**< last used page in file */
-	txnid_t		mm_txnid;			/**< txnid that committed this page */
+	volatile txnid_t	mm_txnid;	/**< txnid that committed this page */
 } MDB_meta;
 
 	/** Buffer for a stack-allocated meta page.
@@ -3601,6 +3601,10 @@ mdb_env_write_meta(MDB_txn *txn)
 		mp->mm_dbs[0] = txn->mt_dbs[0];
 		mp->mm_dbs[1] = txn->mt_dbs[1];
 		mp->mm_last_pg = txn->mt_next_pgno - 1;
+#if !(defined(_MSC_VER) || defined(__i386__) || defined(__x86_64__))
+		/* LY: issue a memory barrier, if not x86. ITS#7969 */
+		__sync_synchronize();
+#endif
 		mp->mm_txnid = txn->mt_txnid;
 		if (!(env->me_flags & (MDB_NOMETASYNC|MDB_NOSYNC))) {
 			unsigned meta_size = env->me_psize;

From bd6d2e6f2c157551f1a0b912716ae92f120361f2 Mon Sep 17 00:00:00 2001
From: "leo@yuriev.ru" <leo@yuriev.ru>
Date: Fri, 17 Oct 2014 22:35:41 +0000
Subject: [PATCH 017/504] ITS#7970 LMDB: Critical Heisenbug

Inconsistent reading & SIGSEGV due to the race condition.
---
 libraries/liblmdb/mdb.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index c84c07f042..0af2047cff 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2558,7 +2558,10 @@ mdb_txn_renew0(MDB_txn *txn)
 					return rc;
 				}
 			}
-			txn->mt_txnid = r->mr_txnid = ti->mti_txnid;
+			do /* LY: Retry on a race, ITS#7970. */
+				r->mr_txnid = ti->mti_txnid;
+			while(r->mr_txnid != ti->mti_txnid);
+			txn->mt_txnid = r->mr_txnid;
 			txn->mt_u.reader = r;
 			meta = env->me_metas[txn->mt_txnid & 1];
 		}

From 9a8eb95674c7b500cfe5f44d03493ff76c9fc0c1 Mon Sep 17 00:00:00 2001
From: "leo@yuriev.ru" <leo@yuriev.ru>
Date: Fri, 17 Oct 2014 22:46:53 +0000
Subject: [PATCH 018/504] ITS#7971 LMDB: clarification in mdb_txn_renew0().

---
 libraries/liblmdb/mdb.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 0af2047cff..b7b2ab3c06 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2543,15 +2543,16 @@ mdb_txn_renew0(MDB_txn *txn)
 					UNLOCK_MUTEX(rmutex);
 					return MDB_READERS_FULL;
 				}
-				ti->mti_readers[i].mr_pid = pid;
-				ti->mti_readers[i].mr_tid = tid;
+				r = &ti->mti_readers[i];
+				r->mr_txnid = (txnid_t)-1;
+				r->mr_tid = tid;
+				r->mr_pid = pid; /* should be written last, see ITS#7971. */
 				if (i == nr)
 					ti->mti_numreaders = ++nr;
 				/* Save numreaders for un-mutexed mdb_env_close() */
 				env->me_numreaders = nr;
 				UNLOCK_MUTEX(rmutex);
 
-				r = &ti->mti_readers[i];
 				new_notls = (env->me_flags & MDB_NOTLS);
 				if (!new_notls && (rc=pthread_setspecific(env->me_txkey, r))) {
 					r->mr_pid = 0;

From 9a72292ac134f80c1befe8102dee1160e57a92ec Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 22 Nov 2014 22:56:31 +0100
Subject: [PATCH 019/504] ITS#7961,#7987 Re-fix txn init.

More fallout from 4d02c741b120786df1b87ee9ed49c1d3f9bc7522:
Don't modify another thread's write txn. Reinit me_txn0 fully.
---
 libraries/liblmdb/mdb.c | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index b7b2ab3c06..3b82277a56 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2584,6 +2584,10 @@ mdb_txn_renew0(MDB_txn *txn)
 		if (txn->mt_txnid == mdb_debug_start)
 			mdb_debug = 1;
 #endif
+		txn->mt_flags = 0;
+		txn->mt_child = NULL;
+		txn->mt_loose_pgs = NULL;
+		txn->mt_loose_count = 0;
 		txn->mt_dirty_room = MDB_IDL_UM_MAX;
 		txn->mt_u.dirty_list = env->me_dirty_list;
 		txn->mt_u.dirty_list[0].mid = 0;
@@ -2669,7 +2673,6 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 	if (!(flags & MDB_RDONLY)) {
 		if (!parent) {
 			txn = env->me_txn0;	/* just reuse preallocated write txn */
-			txn->mt_flags = 0;
 			goto ok;
 		}
 		/* child txns use own copy of cursors */
@@ -2820,6 +2823,8 @@ mdb_txn_reset0(MDB_txn *txn, const char *act)
 		env->me_pghead = NULL;
 		env->me_pglast = 0;
 
+		mdb_cursors_close(txn, 0);
+
 		if (!(env->me_flags & MDB_WRITEMAP)) {
 			mdb_dlist_free(txn);
 		}
@@ -2832,19 +2837,15 @@ mdb_txn_reset0(MDB_txn *txn, const char *act)
 			/* The writer mutex was locked in mdb_txn_begin. */
 			if (env->me_txns)
 				UNLOCK_MUTEX(MDB_MUTEX(env, w));
-		}
-
-		mdb_cursors_close(txn, 0);
-
-		mdb_midl_free(pghead);
-
-		if (txn->mt_parent) {
+		} else {
 			txn->mt_parent->mt_child = NULL;
 			env->me_pgstate = ((MDB_ntxn *)txn)->mnt_pgstate;
 			mdb_midl_free(txn->mt_free_pgs);
 			mdb_midl_free(txn->mt_spill_pgs);
 			free(txn->mt_u.dirty_list);
 		}
+
+		mdb_midl_free(pghead);
 	}
 }
 

From 42110d83a9aa5d1811a77455dcaab7e19ec033e2 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Mon, 1 Dec 2014 08:59:29 +0100
Subject: [PATCH 020/504] Simplify recent changes a bit.

---
 libraries/liblmdb/mdb.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 3b82277a56..51d97d6d31 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2820,11 +2820,8 @@ mdb_txn_reset0(MDB_txn *txn, const char *act)
 		txn->mt_dbxs = NULL;	/* mark txn as reset */
 	} else {
 		pgno_t *pghead = env->me_pghead;
-		env->me_pghead = NULL;
-		env->me_pglast = 0;
 
 		mdb_cursors_close(txn, 0);
-
 		if (!(env->me_flags & MDB_WRITEMAP)) {
 			mdb_dlist_free(txn);
 		}
@@ -2832,6 +2829,9 @@ mdb_txn_reset0(MDB_txn *txn, const char *act)
 		if (!txn->mt_parent) {
 			if (mdb_midl_shrink(&txn->mt_free_pgs))
 				env->me_free_pgs = txn->mt_free_pgs;
+			/* me_pgstate: */
+			env->me_pghead = NULL;
+			env->me_pglast = 0;
 
 			env->me_txn = NULL;
 			/* The writer mutex was locked in mdb_txn_begin. */

From 443a7e40c4280cfd07cbf80c5de62ea072b136e3 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Mon, 1 Dec 2014 08:59:29 +0100
Subject: [PATCH 021/504] Fix robust mutexes - repair mti_txnid.

Broken by e3b6c359a935283c21e1fd7d03b790d87d53b933, if
commit() crashes in write_meta before setting mti_txnid.
---
 libraries/liblmdb/mdb.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 51d97d6d31..9b7f65586a 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -9492,7 +9492,7 @@ static int mdb_reader_check0(MDB_env *env, int rlocked, int *dead)
  */
 static int mdb_mutex_failed(MDB_env *env, mdb_mutex_t *mutex, int rc)
 {
-	int rlocked, rc2;
+	int toggle, rlocked, rc2;
 #ifndef _WIN32
 	enum { WAIT_ABANDONED = EOWNERDEAD };
 #endif
@@ -9502,6 +9502,11 @@ static int mdb_mutex_failed(MDB_env *env, mdb_mutex_t *mutex, int rc)
 		rc = MDB_SUCCESS;
 		rlocked = (mutex == MDB_MUTEX(env, r));
 		if (!rlocked) {
+			/* Keep mti_txnid updated, otherwise next writer can
+			 * overwrite data which latest meta page refers to.
+			 */
+			toggle = mdb_env_pick_meta(env);
+			env->me_txns->mti_txnid = env->me_metas[toggle]->mm_txnid;
 			/* env is hosed if the dead thread was ours */
 			if (env->me_txn) {
 				env->me_flags |= MDB_FATAL_ERROR;

From 4bda1d24617a76569c63d76e8b3fdbdd651e9b13 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Mon, 1 Dec 2014 08:59:29 +0100
Subject: [PATCH 022/504] Clean up SysV semaphore code.

Change MDB_LOCK_FORMAT to differ from Posix semaphores/mutexes.
Do not use array/member length 0 for MDB_txninfo.mt2.
---
 libraries/liblmdb/mdb.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 9b7f65586a..8c548366f2 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -684,18 +684,20 @@ typedef struct MDB_txninfo {
 #define mti_numreaders	mt1.mtb.mtb_numreaders
 		char pad[(sizeof(MDB_txbody)+CACHELINE-1) & ~(CACHELINE-1)];
 	} mt1;
+#ifdef MDB_USE_SYSV_SEM
+#define	mti_semid	mt1.mtb.mtb_semid
+#else
 	union {
 #if defined(_WIN32)
 		char mt2_wmname[MNAME_LEN];
 #define	mti_wmname	mt2.mt2_wmname
-#elif defined(MDB_USE_SYSV_SEM)
-#define	mti_semid	mt1.mtb.mtb_semid
 #else
 		pthread_mutex_t	mt2_wmutex;
 #define mti_wmutex	mt2.mt2_wmutex
 #endif
 		char pad[(MNAME_LEN+CACHELINE-1) & ~(CACHELINE-1)];
 	} mt2;
+#endif
 	MDB_reader	mti_readers[1];
 } MDB_txninfo;
 
@@ -704,6 +706,7 @@ typedef struct MDB_txninfo {
 	((uint32_t) \
 	 ((MDB_LOCK_VERSION) \
 	  /* Flags which describe functionality */ \
+	  + (((MNAME_LEN) == 0)   << 18) /* MDB_USE_SYSV_SEM */ \
 	  + (((MDB_PIDLOCK) != 0) << 16)))
 /** @} */
 

From 4376a19c5c8a740dfb5bb4c2fb17ca303e597aaa Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <h.b.furuseth@usit.uio.no>
Date: Mon, 1 Dec 2014 08:59:29 +0100
Subject: [PATCH 023/504] Fix comments

---
 libraries/liblmdb/mdb.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 8c548366f2..69421c242d 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -935,7 +935,7 @@ typedef struct MDB_meta {
 		/** Stamp identifying this as an LMDB file. It must be set
 		 *	to #MDB_MAGIC. */
 	uint32_t	mm_magic;
-		/** Version number of this lock file. Must be set to #MDB_DATA_VERSION. */
+		/** Version number of this file. Must be set to #MDB_DATA_VERSION. */
 	uint32_t	mm_version;
 	void		*mm_address;		/**< address for fixed mapping */
 	size_t		mm_mapsize;			/**< size of mmap region */
@@ -4097,7 +4097,7 @@ mdb_env_share_locks(MDB_env *env, int *excl)
 	return rc;
 }
 
-/** Try to get exlusive lock, otherwise shared.
+/** Try to get exclusive lock, otherwise shared.
  *	Maintain *excl = -1: no/unknown lock, 0: shared, 1: exclusive.
  */
 static int ESECT
@@ -4238,7 +4238,6 @@ mdb_hash_enc(MDB_val *val, char *encbuf)
  * @param[in] env The LMDB environment.
  * @param[in] lpath The pathname of the file used for the lock region.
  * @param[in] mode The Unix permissions for the file, if we create it.
- * @param[out] excl Resulting file lock type: -1 none, 0 shared, 1 exclusive
  * @param[in,out] excl In -1, out lock type: -1 none, 0 shared, 1 exclusive
  * @return 0 on success, non-zero on failure.
  */

From 0697869d9229ae002b0d4039718812bf5889c2da Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Mon, 1 Dec 2014 08:59:29 +0100
Subject: [PATCH 024/504] mdb_strerror: Silence warning @ Windows

---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 69421c242d..6e9119d56f 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1354,7 +1354,7 @@ mdb_strerror(int err)
 	buf[0] = 0;
 	FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM |
 		FORMAT_MESSAGE_IGNORE_INSERTS,
-		NULL, err, 0, ptr, sizeof(buf), pad);
+		NULL, err, 0, ptr, sizeof(buf), (va_list *)pad);
 	return ptr;
 #else
 	return strerror(err);

From c306423adfdf7c5e5cc1e60323da4fa6b24da49c Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Wed, 3 Dec 2014 12:27:53 +0100
Subject: [PATCH 025/504] Fix WIN32 -> _WIN32

---
 libraries/liblmdb/mdb.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 6e9119d56f..d35d275576 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -200,7 +200,7 @@ union semun {
 #define MDB_DEVEL 0
 #endif
 
-#if defined(WIN32) || (defined(EOWNERDEAD) && !defined(MDB_USE_SYSV_SEM))
+#if defined(_WIN32) || (defined(EOWNERDEAD) && !defined(MDB_USE_SYSV_SEM))
 #define MDB_ROBUST_SUPPORTED	1
 #endif
 
@@ -3881,7 +3881,7 @@ mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers)
 static int ESECT
 mdb_fsize(HANDLE fd, size_t *size)
 {
-#ifdef WIN32
+#ifdef _WIN32
 	LARGE_INTEGER fsize;
 
 	if (!GetFileSizeEx(fd, &fsize))
@@ -4047,7 +4047,7 @@ PIMAGE_TLS_CALLBACK mdb_tls_cbp __attribute__((section (".CRT$XLB"))) = mdb_tls_
 extern const PIMAGE_TLS_CALLBACK mdb_tls_cbp;
 const PIMAGE_TLS_CALLBACK mdb_tls_cbp = mdb_tls_callback;
 #pragma const_seg()
-#else	/* WIN32 */
+#else	/* _WIN32 */
 #pragma comment(linker, "/INCLUDE:__tls_used")
 #pragma comment(linker, "/INCLUDE:_mdb_tls_cbp")
 #pragma data_seg(".CRT$XLB")

From 3e6ac6ef6b983a0bf34a73e14d2fd3bdb1cca777 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <h.b.furuseth@usit.uio.no>
Date: Wed, 3 Dec 2014 12:37:55 +0100
Subject: [PATCH 026/504] For ITS#7789: Ensure mapsize >= pages in use.

Check new mapsizes against mm_last_pg.  Move
mdb_env_init_meta0() so it can set mm_last_pg earlier.
---
 libraries/liblmdb/mdb.c | 35 +++++++++++++++++------------------
 1 file changed, 17 insertions(+), 18 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index d35d275576..7a71e18a0d 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -3507,6 +3507,7 @@ mdb_env_read_header(MDB_env *env, MDB_meta *meta)
 	return 0;
 }
 
+/** Fill in most of the zeroed #MDB_meta for an empty database environment */
 static void ESECT
 mdb_env_init_meta0(MDB_env *env, MDB_meta *meta)
 {
@@ -3523,7 +3524,7 @@ mdb_env_init_meta0(MDB_env *env, MDB_meta *meta)
 
 /** Write the environment parameters of a freshly created DB environment.
  * @param[in] env the environment handle
- * @param[out] meta address of where to store the meta information
+ * @param[in] meta the #MDB_meta to write
  * @return 0 on success, non-zero on failure.
  */
 static int ESECT
@@ -3550,8 +3551,6 @@ mdb_env_init_meta(MDB_env *env, MDB_meta *meta)
 
 	psize = env->me_psize;
 
-	mdb_env_init_meta0(env, meta);
-
 	p = calloc(2, psize);
 	p->mp_pgno = 0;
 	p->mp_flags = P_META;
@@ -3825,16 +3824,16 @@ mdb_env_set_mapsize(MDB_env *env, size_t size)
 	 */
 	if (env->me_map) {
 		int rc;
+		MDB_meta *meta;
 		void *old;
 		if (env->me_txn)
 			return EINVAL;
+		meta = env->me_metas[mdb_env_pick_meta(env)];
 		if (!size)
-			size = env->me_metas[mdb_env_pick_meta(env)]->mm_mapsize;
-		else if (size < env->me_mapsize) {
-			/* If the configured size is smaller, make sure it's
-			 * still big enough. Silently round up to minimum if not.
-			 */
-			size_t minsize = (env->me_metas[mdb_env_pick_meta(env)]->mm_last_pg + 1) * env->me_psize;
+			size = meta->mm_mapsize;
+		{
+			/* Silently round up to minimum if the size is too small */
+			size_t minsize = (meta->mm_last_pg + 1) * env->me_psize;
 			if (size < minsize)
 				size = minsize;
 		}
@@ -3917,8 +3916,6 @@ mdb_env_open2(MDB_env *env)
 		env->me_pidquery = PROCESS_QUERY_INFORMATION;
 #endif /* _WIN32 */
 
-	memset(&meta, 0, sizeof(meta));
-
 	if ((i = mdb_env_read_header(env, &meta)) != 0) {
 		if (i != ENOENT)
 			return i;
@@ -3927,24 +3924,26 @@ mdb_env_open2(MDB_env *env)
 		env->me_psize = env->me_os_psize;
 		if (env->me_psize > MAX_PAGESIZE)
 			env->me_psize = MAX_PAGESIZE;
+		memset(&meta, 0, sizeof(meta));
+		mdb_env_init_meta0(env, &meta);
+		meta.mm_mapsize = DEFAULT_MAPSIZE;
 	} else {
 		env->me_psize = meta.mm_psize;
 	}
 
 	/* Was a mapsize configured? */
 	if (!env->me_mapsize) {
-		/* If this is a new environment, take the default,
-		 * else use the size recorded in the existing env.
-		 */
-		env->me_mapsize = newenv ? DEFAULT_MAPSIZE : meta.mm_mapsize;
-	} else if (env->me_mapsize < meta.mm_mapsize) {
-		/* If the configured size is smaller, make sure it's
-		 * still big enough. Silently round up to minimum if not.
+		env->me_mapsize = meta.mm_mapsize;
+	}
+	{
+		/* Make sure mapsize >= committed data size.  Even when using
+		 * mm_mapsize, which could be broken in old files (ITS#7789).
 		 */
 		size_t minsize = (meta.mm_last_pg + 1) * meta.mm_psize;
 		if (env->me_mapsize < minsize)
 			env->me_mapsize = minsize;
 	}
+	meta.mm_mapsize = env->me_mapsize;
 
 	rc = mdb_env_map(env, (flags & MDB_FIXEDMAP) ? meta.mm_address : NULL);
 	if (rc)

From 1413de3a0474e316d2128ed420a1ebfe0d2b64d8 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Fri, 5 Dec 2014 18:18:53 +0100
Subject: [PATCH 027/504] Fix SysV semaphores - repair mti_txnid.

Similar to 443a7e40c4280cfd07cbf80c5de62ea072b136e3.
---
 libraries/liblmdb/mdb.c | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 7a71e18a0d..a4f13e60f6 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2573,9 +2573,17 @@ mdb_txn_renew0(MDB_txn *txn)
 		if (ti) {
 			if (LOCK_MUTEX(rc, env, MDB_MUTEX(env, w)))
 				return rc;
-
+#ifdef MDB_USE_SYSV_SEM
+			meta = env->me_metas[ mdb_env_pick_meta(env) ];
+			txn->mt_txnid = meta->mm_txnid;
+			/* Update mti_txnid like mdb_mutex_failed() would,
+			 * in case last writer crashed before updating it.
+			 */
+			ti->mti_txnid = txn->mt_txnid;
+#else
 			txn->mt_txnid = ti->mti_txnid;
 			meta = env->me_metas[txn->mt_txnid & 1];
+#endif
 		} else {
 			meta = env->me_metas[ mdb_env_pick_meta(env) ];
 			txn->mt_txnid = meta->mm_txnid;

From b660491d3fa359325cb3615a276bba78f32e2075 Mon Sep 17 00:00:00 2001
From: David Barbour <dmbarbour@gmail.com>
Date: Fri, 5 Dec 2014 11:18:30 -0600
Subject: [PATCH 028/504] ITS#7994 Access to current transaction ID.

I, David Barbour, hereby place the following modifications to OpenLDAP
Software (and only these modifications) into the public domain. Hence,
these modifications may be freely used and/or redistributed for any
purpose with or without attribution and/or other notice.
---
 libraries/liblmdb/lmdb.h | 18 +++++++++++++++++-
 libraries/liblmdb/mdb.c  |  7 +++++++
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 4236218bb5..803b28618b 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -226,6 +226,9 @@ typedef struct MDB_env MDB_env;
  */
 typedef struct MDB_txn MDB_txn;
 
+/** @brief Unique identifier for an active or recent transaction. */
+typedef size_t MDB_txnid_t;
+
 /** @brief A handle for an individual database in the DB environment. */
 typedef unsigned int	MDB_dbi;
 
@@ -448,7 +451,7 @@ typedef struct MDB_envinfo {
 	void	*me_mapaddr;			/**< Address of map, if fixed */
 	size_t	me_mapsize;				/**< Size of the data memory map */
 	size_t	me_last_pgno;			/**< ID of the last used page */
-	size_t	me_last_txnid;			/**< ID of the last committed transaction */
+	MDB_txnid_t  me_last_txnid;		/**< ID of the last committed transaction */
 	unsigned int me_maxreaders;		/**< max reader slots in the environment */
 	unsigned int me_numreaders;		/**< max reader slots used in the environment */
 } MDB_envinfo;
@@ -950,6 +953,19 @@ int  mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **
 	 */
 MDB_env *mdb_txn_env(MDB_txn *txn);
 
+	/** @brief Return the transaction's #MDB_txnid_t
+	 *
+	 * This returns the identifier associated with this transaction. For a
+	 * read-only transaction, this corresponds to the snapshot being read;
+	 * concurrent readers will frequently have the same transaction ID. For
+	 * a write transaction, this is always the snapshot read plus one. When
+	 * a write transaction aborts, the next transaction ID will be reused.
+	 *
+	 * @param[in] txn A transaction handle returned by #mdb_txn_begin()
+	 * @return A transaction ID, valid if input is an active transaction.
+	 */
+MDB_txnid_t mdb_txn_id(MDB_txn *txn);
+
 	/** @brief Commit all the operations of a transaction into the database.
 	 *
 	 * The transaction handle is freed. It and its cursors must not be used
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index a4f13e60f6..87aa4f8cee 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2775,6 +2775,13 @@ mdb_txn_env(MDB_txn *txn)
 	return txn->mt_env;
 }
 
+MDB_txnid_t
+mdb_txn_id(MDB_txn *txn)
+{
+    if(!txn) return (txnid_t)-1;
+    return txn->mt_txnid;
+}
+
 /** Export or close DBI handles opened in this txn. */
 static void
 mdb_dbis_update(MDB_txn *txn, int keep)

From c36c167cc49a59d4f40ae5fc259c013de601164e Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Fri, 5 Dec 2014 19:15:41 +0000
Subject: [PATCH 029/504] ITS#7994 fix prev commit

Strip docs promising behaviors that we don't promise to hold.
Strip MDB_txnid_t typedef.
---
 libraries/liblmdb/lmdb.h | 13 ++++---------
 libraries/liblmdb/mdb.c  |  4 ++--
 2 files changed, 6 insertions(+), 11 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 803b28618b..621a8f6e23 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -226,9 +226,6 @@ typedef struct MDB_env MDB_env;
  */
 typedef struct MDB_txn MDB_txn;
 
-/** @brief Unique identifier for an active or recent transaction. */
-typedef size_t MDB_txnid_t;
-
 /** @brief A handle for an individual database in the DB environment. */
 typedef unsigned int	MDB_dbi;
 
@@ -451,7 +448,7 @@ typedef struct MDB_envinfo {
 	void	*me_mapaddr;			/**< Address of map, if fixed */
 	size_t	me_mapsize;				/**< Size of the data memory map */
 	size_t	me_last_pgno;			/**< ID of the last used page */
-	MDB_txnid_t  me_last_txnid;		/**< ID of the last committed transaction */
+	size_t  me_last_txnid;		/**< ID of the last committed transaction */
 	unsigned int me_maxreaders;		/**< max reader slots in the environment */
 	unsigned int me_numreaders;		/**< max reader slots used in the environment */
 } MDB_envinfo;
@@ -953,18 +950,16 @@ int  mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **
 	 */
 MDB_env *mdb_txn_env(MDB_txn *txn);
 
-	/** @brief Return the transaction's #MDB_txnid_t
+	/** @brief Return the transaction's ID.
 	 *
 	 * This returns the identifier associated with this transaction. For a
 	 * read-only transaction, this corresponds to the snapshot being read;
-	 * concurrent readers will frequently have the same transaction ID. For
-	 * a write transaction, this is always the snapshot read plus one. When
-	 * a write transaction aborts, the next transaction ID will be reused.
+	 * concurrent readers will frequently have the same transaction ID.
 	 *
 	 * @param[in] txn A transaction handle returned by #mdb_txn_begin()
 	 * @return A transaction ID, valid if input is an active transaction.
 	 */
-MDB_txnid_t mdb_txn_id(MDB_txn *txn);
+size_t mdb_txn_id(MDB_txn *txn);
 
 	/** @brief Commit all the operations of a transaction into the database.
 	 *
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 87aa4f8cee..f61bbdf6ed 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2775,10 +2775,10 @@ mdb_txn_env(MDB_txn *txn)
 	return txn->mt_env;
 }
 
-MDB_txnid_t
+size_t
 mdb_txn_id(MDB_txn *txn)
 {
-    if(!txn) return (txnid_t)-1;
+    if(!txn) return 0;
     return txn->mt_txnid;
 }
 

From 1a7243b3f41ba1896b0ee69be5498f769cd2cfe5 Mon Sep 17 00:00:00 2001
From: Leo Yuriev <leo@yuriev.ru>
Date: Fri, 5 Dec 2014 19:30:31 +0000
Subject: [PATCH 030/504] ITS#7987 fix excessive space for single write txn

---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index f61bbdf6ed..1127dd0e3d 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -4603,7 +4603,7 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 		if (!(flags & MDB_RDONLY)) {
 			MDB_txn *txn;
 			int tsize = sizeof(MDB_txn), size = tsize + env->me_maxdbs *
-				(sizeof(MDB_db)+sizeof(MDB_cursor)+sizeof(unsigned int)+1);
+				(sizeof(MDB_db)+sizeof(MDB_cursor *)+sizeof(unsigned int)+1);
 			txn = calloc(1, size);
 			if (txn) {
 				txn->mt_dbs = (MDB_db *)((char *)txn + tsize);

From 03ddbcf37de9ebdf244d5be4d2213852b0fd6abb Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Fri, 5 Dec 2014 21:00:21 +0100
Subject: [PATCH 031/504] C90-compatible code for SysV semaphores

---
 libraries/liblmdb/mdb.c | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 1127dd0e3d..32e24c00f9 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -270,13 +270,18 @@ typedef struct mdb_mutex {
 
 #define MDB_MUTEX(env, rw)		(&(env)->me_##rw##mutex)
 #define LOCK_MUTEX0(mutex)		mdb_sem_wait(mutex)
-#define UNLOCK_MUTEX(mutex)		do { struct sembuf sb = { mutex->semnum, 1, SEM_UNDO }; semop(mutex->semid, &sb, 1); } while(0)
+#define UNLOCK_MUTEX(mutex)		do { \
+	struct sembuf sb = { 0, 1, SEM_UNDO }; \
+	sb.sem_num = (mutex)->semnum; \
+	semop((mutex)->semid, &sb, 1); \
+} while(0)
 
 static int
 mdb_sem_wait(mdb_mutex_t *sem)
 {
    int rc;
-   struct sembuf sb = { sem->semnum, -1, SEM_UNDO };
+   struct sembuf sb = { 0, -1, SEM_UNDO };
+   sb.sem_num = sem->semnum;
    while ((rc = semop(sem->semid, &sb, 1)) && (rc = errno) == EINTR) ;
    return rc;
 }
@@ -4417,6 +4422,11 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 		env->me_txns->mti_numreaders = 0;
 
 	} else {
+#ifdef MDB_USE_SYSV_SEM
+		struct semid_ds buf;
+		union semun semu;
+		int semid;
+#endif
 		if (env->me_txns->mti_magic != MDB_MAGIC) {
 			DPUTS("lock region has invalid magic");
 			rc = MDB_INVALID;
@@ -4438,9 +4448,7 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 		env->me_wmutex = OpenMutex(SYNCHRONIZE, FALSE, env->me_txns->mti_wmname);
 		if (!env->me_wmutex) goto fail_errno;
 #elif defined(MDB_USE_SYSV_SEM)
-		struct semid_ds buf;
-		union semun semu;
-		int semid = env->me_txns->mti_semid;
+		semid = env->me_txns->mti_semid;
 		semu.buf = &buf;
 
 		/* check for read access */

From 9cc04f604f80f033d712f5f1faeb4ed97ca74f40 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 8 Dec 2014 03:21:09 +0000
Subject: [PATCH 032/504] Clarify mdb_dbi_open doc

The concurrency restriction is on threads within a single process.
Multiple processes can use mdb_dbi_open without affecting each other.
---
 libraries/liblmdb/lmdb.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 621a8f6e23..ad0ebed10a 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -1042,9 +1042,9 @@ int  mdb_txn_renew(MDB_txn *txn);
 	 * After a successful commit the
 	 * handle will reside in the shared environment, and may be used
 	 * by other transactions. This function must not be called from
-	 * multiple concurrent transactions. A transaction that uses this function
-	 * must finish (either commit or abort) before any other transaction may
-	 * use this function.
+	 * multiple concurrent transactions in the same process. A transaction
+	 * that uses this function must finish (either commit or abort) before
+	 * any other transaction in the process may use this function.
 	 *
 	 * To use named databases (with name != NULL), #mdb_env_set_maxdbs()
 	 * must be called before opening the environment.  Database names

From cccc947b4ab1139d808eee01df2b828e3f0183ee Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Mon, 8 Dec 2014 09:56:36 +0100
Subject: [PATCH 033/504] Try to avoid an invalid datafile after failed init

---
 libraries/liblmdb/mdb.c | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 32e24c00f9..cc6eaa20bb 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -3965,6 +3965,20 @@ mdb_env_open2(MDB_env *env)
 	}
 	meta.mm_mapsize = env->me_mapsize;
 
+	if (newenv && !(flags & MDB_FIXEDMAP)) {
+		/* mdb_env_map() may grow the datafile.  Write the metapages
+		 * first, so the file will be valid if initialization fails.
+		 * Except with FIXEDMAP, since we do not yet know mm_address.
+		 * We could fill in mm_address later, but then a different
+		 * program might end up doing that - one with a memory layout
+		 * and map address which does not suit the main program.
+		 */
+		rc = mdb_env_init_meta(env, &meta);
+		if (rc)
+			return rc;
+		newenv = 0;
+	}
+
 	rc = mdb_env_map(env, (flags & MDB_FIXEDMAP) ? meta.mm_address : NULL);
 	if (rc)
 		return rc;

From 6c7117325892a68e28141660cb9344e2ccef380a Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 13 Dec 2014 20:35:08 +0100
Subject: [PATCH 034/504] make clean: remove *.lo

---
 libraries/liblmdb/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile
index 5f8ef64461..bca5cd3819 100644
--- a/libraries/liblmdb/Makefile
+++ b/libraries/liblmdb/Makefile
@@ -41,7 +41,7 @@ install: $(ILIBS) $(IPROGS) $(IHDRS)
 	for f in $(IDOCS); do cp $$f $(DESTDIR)$(prefix)/man/man1; done
 
 clean:
-	rm -rf $(PROGS) *.[ao] *.so *~ testdb
+	rm -rf $(PROGS) *.[ao] *.[ls]o *~ testdb
 
 test:	all
 	rm -rf testdb && mkdir testdb

From 7ce29b9edbdaf34b7aeb545324008ed4dff62952 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 15 Dec 2014 20:40:59 +0000
Subject: [PATCH 035/504] Fix MDB_APPEND doc

We actually compare the new key against the last key, to prevent
corruption.
---
 libraries/liblmdb/lmdb.h | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index ad0ebed10a..f318ad416d 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -1287,10 +1287,9 @@ int  mdb_get(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data);
 	 *		LMDB does nothing else with this memory, the caller is expected
 	 *		to modify all of the space requested.
 	 *	<li>#MDB_APPEND - append the given key/data pair to the end of the
-	 *		database. No key comparisons are performed. This option allows
-	 *		fast bulk loading when keys are already known to be in the
-	 *		correct order. Loading unsorted keys with this flag will cause
-	 *		data corruption.
+	 *		database. This option allows fast bulk loading when keys are
+	 *		already known to be in the correct order. Loading unsorted keys
+	 *		with this flag will cause a #MDB_KEYEXIST error.
 	 *	<li>#MDB_APPENDDUP - as above, but for sorted dup data.
 	 * </ul>
 	 * @return A non-zero error value on failure and 0 on success. Some possible

From 0018eeb2c3b2239c30def9d47c9d194a4ebf35fe Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 18 Dec 2014 04:38:53 +0000
Subject: [PATCH 036/504] Hack for potential ext3/ext4 corruption issue

Use regular fsync() if we think this commit grew the DB file.
---
 libraries/liblmdb/mdb.c | 43 ++++++++++++++++++++++++++++++++++++-----
 1 file changed, 38 insertions(+), 5 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index cc6eaa20bb..d9ab5bd383 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -368,6 +368,7 @@ static int mdb_mutex_failed(MDB_env *env, mdb_mutex_t *mutex, int rc);
  */
 #ifndef MDB_FDATASYNC
 # define MDB_FDATASYNC	fdatasync
+# define HAVE_FDATASYNC	1
 #endif
 
 #ifndef MDB_MSYNC
@@ -1154,7 +1155,7 @@ struct MDB_env {
 	MDB_txn		*me_txn;		/**< current write transaction */
 	MDB_txn		*me_txn0;		/**< prealloc'd write transaction */
 	size_t		me_mapsize;		/**< size of the data memory map */
-	off_t		me_size;		/**< current file size */
+	size_t		me_size;		/**< current file size */
 	pgno_t		me_maxpg;		/**< me_mapsize / me_psize */
 	MDB_dbx		*me_dbxs;		/**< array of static DB info */
 	uint16_t	*me_dbflags;	/**< array of flags from MDB_db.md_flags */
@@ -2341,10 +2342,19 @@ fail:
 	return rc;
 }
 
-int
-mdb_env_sync(MDB_env *env, int force)
+/* internal env_sync flags: */
+#define FORCE	1		/* as before, force a flush */
+#define FGREW	0x8000	/* file has grown, do a full fsync instead of just
+	   fdatasync. We shouldn't have to do this, according to the POSIX spec.
+	   But common Linux FSs violate the spec and won't sync required metadata
+	   correctly when the file grows. This only makes a difference if the
+	   platform actually distinguishes fdatasync from fsync.
+	   http://www.openldap.org/lists/openldap-devel/201411/msg00000.html */
+
+static int
+mdb_env_sync0(MDB_env *env, int flag)
 {
-	int rc = 0;
+	int rc = 0, force = flag & FORCE;
 	if (force || !F_ISSET(env->me_flags, MDB_NOSYNC)) {
 		if (env->me_flags & MDB_WRITEMAP) {
 			int flags = ((env->me_flags & MDB_MAPASYNC) && !force)
@@ -2356,6 +2366,12 @@ mdb_env_sync(MDB_env *env, int force)
 				rc = ErrCode();
 #endif
 		} else {
+#ifdef HAVE_FDATASYNC
+			if (flag & FGREW) {
+				if (fsync(env->me_fd))	/* Avoid ext-fs bugs, do full sync */
+					rc = ErrCode();
+			} else
+#endif
 			if (MDB_FDATASYNC(env->me_fd))
 				rc = ErrCode();
 		}
@@ -2363,6 +2379,12 @@ mdb_env_sync(MDB_env *env, int force)
 	return rc;
 }
 
+int
+mdb_env_sync(MDB_env *env, int force)
+{
+	return mdb_env_sync0(env, force != 0);
+}
+
 /** Back up parent txn's cursors, then grab the originals for tracking */
 static int
 mdb_cursor_shadow(MDB_txn *src, MDB_txn *dst)
@@ -3437,8 +3459,15 @@ mdb_txn_commit(MDB_txn *txn)
 	mdb_audit(txn);
 #endif
 
+	i = 0;
+#ifdef HAVE_FDATASYNC
+	if (txn->mt_next_pgno * env->me_psize > env->me_size) {
+		i |= FGREW;
+		env->me_size = txn->mt_next_pgno * env->me_psize;
+	}
+#endif
 	if ((rc = mdb_page_flush(txn, 0)) ||
-		(rc = mdb_env_sync(env, 0)) ||
+		(rc = mdb_env_sync(env, i)) ||
 		(rc = mdb_env_write_meta(txn)))
 		goto fail;
 
@@ -3979,6 +4008,10 @@ mdb_env_open2(MDB_env *env)
 		newenv = 0;
 	}
 
+	rc = mdb_fsize(env->me_fd, &env->me_size);
+	if (rc)
+		return rc;
+
 	rc = mdb_env_map(env, (flags & MDB_FIXEDMAP) ? meta.mm_address : NULL);
 	if (rc)
 		return rc;

From 985bbbbdd5d64e57f55249ffdeb7c08035b240b2 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sun, 21 Dec 2014 00:13:50 +0000
Subject: [PATCH 037/504] Fix prev commit for env_sync0

---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index d9ab5bd383..002d88bd3e 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -3467,7 +3467,7 @@ mdb_txn_commit(MDB_txn *txn)
 	}
 #endif
 	if ((rc = mdb_page_flush(txn, 0)) ||
-		(rc = mdb_env_sync(env, i)) ||
+		(rc = mdb_env_sync0(env, i)) ||
 		(rc = mdb_env_write_meta(txn)))
 		goto fail;
 

From 0ef1e0b16d4366d223601d6c7da35dc8c208a9b6 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 8 Jan 2015 11:28:33 +0000
Subject: [PATCH 038/504] Revert "Fix prev commit for env_sync0"

This reverts commit 985bbbbdd5d64e57f55249ffdeb7c08035b240b2.

Revert "Hack for potential ext3/ext4 corruption issue"

This reverts commit 0018eeb2c3b2239c30def9d47c9d194a4ebf35fe.
---
 libraries/liblmdb/mdb.c | 43 +++++------------------------------------
 1 file changed, 5 insertions(+), 38 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 002d88bd3e..cc6eaa20bb 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -368,7 +368,6 @@ static int mdb_mutex_failed(MDB_env *env, mdb_mutex_t *mutex, int rc);
  */
 #ifndef MDB_FDATASYNC
 # define MDB_FDATASYNC	fdatasync
-# define HAVE_FDATASYNC	1
 #endif
 
 #ifndef MDB_MSYNC
@@ -1155,7 +1154,7 @@ struct MDB_env {
 	MDB_txn		*me_txn;		/**< current write transaction */
 	MDB_txn		*me_txn0;		/**< prealloc'd write transaction */
 	size_t		me_mapsize;		/**< size of the data memory map */
-	size_t		me_size;		/**< current file size */
+	off_t		me_size;		/**< current file size */
 	pgno_t		me_maxpg;		/**< me_mapsize / me_psize */
 	MDB_dbx		*me_dbxs;		/**< array of static DB info */
 	uint16_t	*me_dbflags;	/**< array of flags from MDB_db.md_flags */
@@ -2342,19 +2341,10 @@ fail:
 	return rc;
 }
 
-/* internal env_sync flags: */
-#define FORCE	1		/* as before, force a flush */
-#define FGREW	0x8000	/* file has grown, do a full fsync instead of just
-	   fdatasync. We shouldn't have to do this, according to the POSIX spec.
-	   But common Linux FSs violate the spec and won't sync required metadata
-	   correctly when the file grows. This only makes a difference if the
-	   platform actually distinguishes fdatasync from fsync.
-	   http://www.openldap.org/lists/openldap-devel/201411/msg00000.html */
-
-static int
-mdb_env_sync0(MDB_env *env, int flag)
+int
+mdb_env_sync(MDB_env *env, int force)
 {
-	int rc = 0, force = flag & FORCE;
+	int rc = 0;
 	if (force || !F_ISSET(env->me_flags, MDB_NOSYNC)) {
 		if (env->me_flags & MDB_WRITEMAP) {
 			int flags = ((env->me_flags & MDB_MAPASYNC) && !force)
@@ -2366,12 +2356,6 @@ mdb_env_sync0(MDB_env *env, int flag)
 				rc = ErrCode();
 #endif
 		} else {
-#ifdef HAVE_FDATASYNC
-			if (flag & FGREW) {
-				if (fsync(env->me_fd))	/* Avoid ext-fs bugs, do full sync */
-					rc = ErrCode();
-			} else
-#endif
 			if (MDB_FDATASYNC(env->me_fd))
 				rc = ErrCode();
 		}
@@ -2379,12 +2363,6 @@ mdb_env_sync0(MDB_env *env, int flag)
 	return rc;
 }
 
-int
-mdb_env_sync(MDB_env *env, int force)
-{
-	return mdb_env_sync0(env, force != 0);
-}
-
 /** Back up parent txn's cursors, then grab the originals for tracking */
 static int
 mdb_cursor_shadow(MDB_txn *src, MDB_txn *dst)
@@ -3459,15 +3437,8 @@ mdb_txn_commit(MDB_txn *txn)
 	mdb_audit(txn);
 #endif
 
-	i = 0;
-#ifdef HAVE_FDATASYNC
-	if (txn->mt_next_pgno * env->me_psize > env->me_size) {
-		i |= FGREW;
-		env->me_size = txn->mt_next_pgno * env->me_psize;
-	}
-#endif
 	if ((rc = mdb_page_flush(txn, 0)) ||
-		(rc = mdb_env_sync0(env, i)) ||
+		(rc = mdb_env_sync(env, 0)) ||
 		(rc = mdb_env_write_meta(txn)))
 		goto fail;
 
@@ -4008,10 +3979,6 @@ mdb_env_open2(MDB_env *env)
 		newenv = 0;
 	}
 
-	rc = mdb_fsize(env->me_fd, &env->me_size);
-	if (rc)
-		return rc;
-
 	rc = mdb_env_map(env, (flags & MDB_FIXEDMAP) ? meta.mm_address : NULL);
 	if (rc)
 		return rc;

From 9585c012335b2a28fbfdd267bcdeeeea91dc0f08 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 8 Jan 2015 11:42:08 +0000
Subject: [PATCH 039/504] Simpler fdatasync hack

---
 libraries/liblmdb/mdb.c | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index cc6eaa20bb..16530b9ac0 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -368,6 +368,12 @@ static int mdb_mutex_failed(MDB_env *env, mdb_mutex_t *mutex, int rc);
  */
 #ifndef MDB_FDATASYNC
 # define MDB_FDATASYNC	fdatasync
+# ifndef MDB_SAFE_FDATASYNC
+/** Linux ext3fs and ext4fs don't implement fdatasync correctly
+ *	on older kernels. xfs is known to be safe. https://lkml.org/lkml/2012/9/3/83
+ */
+#  define	MDB_BROKEN_FDATASYNC
+# endif
 #endif
 
 #ifndef MDB_MSYNC
@@ -1154,7 +1160,7 @@ struct MDB_env {
 	MDB_txn		*me_txn;		/**< current write transaction */
 	MDB_txn		*me_txn0;		/**< prealloc'd write transaction */
 	size_t		me_mapsize;		/**< size of the data memory map */
-	off_t		me_size;		/**< current file size */
+	size_t		me_size;		/**< current file size */
 	pgno_t		me_maxpg;		/**< me_mapsize / me_psize */
 	MDB_dbx		*me_dbxs;		/**< array of static DB info */
 	uint16_t	*me_dbflags;	/**< array of flags from MDB_db.md_flags */
@@ -2341,6 +2347,10 @@ fail:
 	return rc;
 }
 
+#ifdef MDB_BROKEN_FDATASYNC
+static int ESECT mdb_fsize(HANDLE fd, size_t *size);
+#endif
+
 int
 mdb_env_sync(MDB_env *env, int force)
 {
@@ -2356,6 +2366,15 @@ mdb_env_sync(MDB_env *env, int force)
 				rc = ErrCode();
 #endif
 		} else {
+#ifdef MDB_BROKEN_FDATASYNC
+			size_t sz = 0;
+			if (mdb_fsize(env->me_fd, &sz) != MDB_SUCCESS || sz != env->me_size) {
+				if (fsync(env->me_fd))
+					rc = ErrCode();
+				else if (sz)
+					env->me_size = sz;
+			} else
+#endif
 			if (MDB_FDATASYNC(env->me_fd))
 				rc = ErrCode();
 		}

From 293d6bb47f7f0d54a45d57ce2be3a9595e806dc0 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 8 Jan 2015 11:43:57 +0000
Subject: [PATCH 040/504] Note MDB_SAFE_FDATASYNC

---
 libraries/liblmdb/Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile
index bca5cd3819..90e4e21457 100644
--- a/libraries/liblmdb/Makefile
+++ b/libraries/liblmdb/Makefile
@@ -11,6 +11,7 @@
 # - MDB_USE_POSIX_SEM
 # - MDB_DSYNC
 # - MDB_FDATASYNC
+# - MDB_SAFE_FDATASYNC
 # - MDB_USE_PWRITEV
 #
 # There may be other macros in mdb.c of interest. You should

From e86072a9c79d7fb5fdf63ffbe7b7a36731f0a11a Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 8 Jan 2015 12:17:29 +0000
Subject: [PATCH 041/504] Revert "Note MDB_SAFE_FDATASYNC"

This reverts commit 293d6bb47f7f0d54a45d57ce2be3a9595e806dc0.

Revert "Simpler fdatasync hack"

This reverts commit 9585c012335b2a28fbfdd267bcdeeeea91dc0f08.
---
 libraries/liblmdb/Makefile |  1 -
 libraries/liblmdb/mdb.c    | 21 +--------------------
 2 files changed, 1 insertion(+), 21 deletions(-)

diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile
index 90e4e21457..bca5cd3819 100644
--- a/libraries/liblmdb/Makefile
+++ b/libraries/liblmdb/Makefile
@@ -11,7 +11,6 @@
 # - MDB_USE_POSIX_SEM
 # - MDB_DSYNC
 # - MDB_FDATASYNC
-# - MDB_SAFE_FDATASYNC
 # - MDB_USE_PWRITEV
 #
 # There may be other macros in mdb.c of interest. You should
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 16530b9ac0..cc6eaa20bb 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -368,12 +368,6 @@ static int mdb_mutex_failed(MDB_env *env, mdb_mutex_t *mutex, int rc);
  */
 #ifndef MDB_FDATASYNC
 # define MDB_FDATASYNC	fdatasync
-# ifndef MDB_SAFE_FDATASYNC
-/** Linux ext3fs and ext4fs don't implement fdatasync correctly
- *	on older kernels. xfs is known to be safe. https://lkml.org/lkml/2012/9/3/83
- */
-#  define	MDB_BROKEN_FDATASYNC
-# endif
 #endif
 
 #ifndef MDB_MSYNC
@@ -1160,7 +1154,7 @@ struct MDB_env {
 	MDB_txn		*me_txn;		/**< current write transaction */
 	MDB_txn		*me_txn0;		/**< prealloc'd write transaction */
 	size_t		me_mapsize;		/**< size of the data memory map */
-	size_t		me_size;		/**< current file size */
+	off_t		me_size;		/**< current file size */
 	pgno_t		me_maxpg;		/**< me_mapsize / me_psize */
 	MDB_dbx		*me_dbxs;		/**< array of static DB info */
 	uint16_t	*me_dbflags;	/**< array of flags from MDB_db.md_flags */
@@ -2347,10 +2341,6 @@ fail:
 	return rc;
 }
 
-#ifdef MDB_BROKEN_FDATASYNC
-static int ESECT mdb_fsize(HANDLE fd, size_t *size);
-#endif
-
 int
 mdb_env_sync(MDB_env *env, int force)
 {
@@ -2366,15 +2356,6 @@ mdb_env_sync(MDB_env *env, int force)
 				rc = ErrCode();
 #endif
 		} else {
-#ifdef MDB_BROKEN_FDATASYNC
-			size_t sz = 0;
-			if (mdb_fsize(env->me_fd, &sz) != MDB_SUCCESS || sz != env->me_size) {
-				if (fsync(env->me_fd))
-					rc = ErrCode();
-				else if (sz)
-					env->me_size = sz;
-			} else
-#endif
 			if (MDB_FDATASYNC(env->me_fd))
 				rc = ErrCode();
 		}

From 462dc097451834477b597447af69c5acc93182b7 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 8 Jan 2015 12:54:14 +0000
Subject: [PATCH 042/504] fdatasync hack, again

Check for ext3/ext4 fs, then check kernel version.
---
 libraries/liblmdb/mdb.c | 62 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 62 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index cc6eaa20bb..b6703d6857 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1180,6 +1180,9 @@ struct MDB_env {
 #ifdef _WIN32
 	int		me_pidquery;		/**< Used in OpenProcess */
 #endif
+#ifdef __linux
+	int		me_fsynconly;		/**< fdatasync is unreliable */
+#endif
 #if defined(_WIN32) || defined(MDB_USE_SYSV_SEM)
 	/* Windows mutexes/SysV semaphores do not reside in shared mem */
 	mdb_mutex_t	me_rmutex;
@@ -2356,6 +2359,12 @@ mdb_env_sync(MDB_env *env, int force)
 				rc = ErrCode();
 #endif
 		} else {
+#ifdef __linux
+			if (env->me_fsynconly) {
+				if (fsync(env->me_fd))
+					rc = ErrCode();
+			} else
+#endif
 			if (MDB_FDATASYNC(env->me_fd))
 				rc = ErrCode();
 		}
@@ -3918,6 +3927,11 @@ mdb_fsize(HANDLE fd, size_t *size)
 	return MDB_SUCCESS;
 }
 
+#ifdef __linux
+#include <sys/utsname.h>
+#include <sys/vfs.h>
+#endif
+
 /** Further setup required for opening an LMDB environment
  */
 static int ESECT
@@ -3936,6 +3950,54 @@ mdb_env_open2(MDB_env *env)
 		env->me_pidquery = PROCESS_QUERY_INFORMATION;
 #endif /* _WIN32 */
 
+#ifdef __linux
+	/* ext3/ext4 fdatasync is broken on some older Linux kernels.
+	 * https://lkml.org/lkml/2012/9/3/83
+	 * Kernels after 3.6-rc6 are known good.
+	 * https://lkml.org/lkml/2012/9/10/556
+	 * See if the DB is on ext3/ext4, then check for new enough kernel
+	 * Kernels 2.6.32.60, 2.6.34.15, 3.2.30, and 3.5.4 are also known
+	 * to be patched.
+	 */
+	{
+		struct statfs st;
+		fstatfs(env->me_fd, &st);
+		while (st.f_type == 0xEF53) {
+			struct utsname uts;
+			int i;
+			uname(&uts);
+			if (uts.release[0] < '3') {
+				if (!strncmp(uts.release, "2.6.32.", 7)) {
+					i = atoi(uts.release+7);
+					if (i >= 60)
+						break;	/* 2.6.32.60 and newer is OK */
+				} else if (!strncmp(uts.release, "2.6.34.", 7)) {
+					i = atoi(uts.release+7);
+					if (i >= 15)
+						break;	/* 2.6.34.15 and newer is OK */
+				}
+			} else if (uts.release[0] == '3') {
+				i = atoi(uts.release+2);
+				if (i > 5)
+					break;	/* 3.6 and newer is OK */
+				if (i == 5) {
+					i = atoi(uts.release+4);
+					if (i >= 4)
+						break;	/* 3.5.4 and newer is OK */
+				} else if (i == 2) {
+					i = atoi(uts.release+4);
+					if (i >= 30)
+						break;	/* 3.2.30 and newer is OK */
+				}
+			} else {	/* 4.x and newer is OK */
+				break;
+			}
+			env->me_fsynconly = 1;
+			break;
+		}
+	}
+#endif
+
 	if ((i = mdb_env_read_header(env, &meta)) != 0) {
 		if (i != ENOENT)
 			return i;

From f83f62a7aeb3a873564b3fba53132d026d1f8644 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Fri, 9 Jan 2015 11:25:07 +0000
Subject: [PATCH 043/504] ITS#8021 env_sync is invalid in RDONLY env

---
 libraries/liblmdb/lmdb.h | 4 +++-
 libraries/liblmdb/mdb.c  | 2 ++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index f318ad416d..d22009bbcc 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -703,7 +703,8 @@ int  mdb_env_info(MDB_env *env, MDB_envinfo *stat);
 	 * Data is always written to disk when #mdb_txn_commit() is called,
 	 * but the operating system may keep it buffered. LMDB always flushes
 	 * the OS buffers upon commit as well, unless the environment was
-	 * opened with #MDB_NOSYNC or in part #MDB_NOMETASYNC.
+	 * opened with #MDB_NOSYNC or in part #MDB_NOMETASYNC. This call is
+	 * not valid if the environment was opened with #MDB_RDONLY.
 	 * @param[in] env An environment handle returned by #mdb_env_create()
 	 * @param[in] force If non-zero, force a synchronous flush.  Otherwise
 	 *  if the environment has the #MDB_NOSYNC flag set the flushes
@@ -711,6 +712,7 @@ int  mdb_env_info(MDB_env *env, MDB_envinfo *stat);
 	 * @return A non-zero error value on failure and 0 on success. Some possible
 	 * errors are:
 	 * <ul>
+	 *	<li>EACCES - the environment is read-only.
 	 *	<li>EINVAL - an invalid parameter was specified.
 	 *	<li>EIO - an error occurred during synchronization.
 	 * </ul>
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index b6703d6857..121c53c3d9 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2348,6 +2348,8 @@ int
 mdb_env_sync(MDB_env *env, int force)
 {
 	int rc = 0;
+	if (env->me_flags & MDB_RDONLY)
+		return EACCES;
 	if (force || !F_ISSET(env->me_flags, MDB_NOSYNC)) {
 		if (env->me_flags & MDB_WRITEMAP) {
 			int flags = ((env->me_flags & MDB_MAPASYNC) && !force)

From ea89e3d269bd68cb3ab494f0dcf0b379574265c5 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sun, 11 Jan 2015 11:37:37 +0000
Subject: [PATCH 044/504] Tweak conditionals for fdatasync hack

---
 libraries/liblmdb/Makefile |  1 +
 libraries/liblmdb/mdb.c    | 14 ++++++++++----
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile
index bca5cd3819..2d0983eff0 100644
--- a/libraries/liblmdb/Makefile
+++ b/libraries/liblmdb/Makefile
@@ -11,6 +11,7 @@
 # - MDB_USE_POSIX_SEM
 # - MDB_DSYNC
 # - MDB_FDATASYNC
+# - MDB_FDATASYNC_WORKS
 # - MDB_USE_PWRITEV
 #
 # There may be other macros in mdb.c of interest. You should
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 121c53c3d9..d82c3f022b 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -79,6 +79,12 @@ extern int cacheflush(char *addr, int nbytes, int cache);
 #define CACHEFLUSH(addr, bytes, cache)
 #endif
 
+#if defined(__linux) && !defined(MDB_FDATASYNC_WORKS)
+/** fdatasync is broken on ext3/ext4fs on older kernels, see
+ *	description in #mdb_env_open2 comments
+ */
+#define	BROKEN_FDATASYNC
+#endif
 
 #include <errno.h>
 #include <limits.h>
@@ -1180,7 +1186,7 @@ struct MDB_env {
 #ifdef _WIN32
 	int		me_pidquery;		/**< Used in OpenProcess */
 #endif
-#ifdef __linux
+#ifdef BROKEN_FDATASYNC
 	int		me_fsynconly;		/**< fdatasync is unreliable */
 #endif
 #if defined(_WIN32) || defined(MDB_USE_SYSV_SEM)
@@ -2361,7 +2367,7 @@ mdb_env_sync(MDB_env *env, int force)
 				rc = ErrCode();
 #endif
 		} else {
-#ifdef __linux
+#ifdef BROKEN_FDATASYNC
 			if (env->me_fsynconly) {
 				if (fsync(env->me_fd))
 					rc = ErrCode();
@@ -3929,7 +3935,7 @@ mdb_fsize(HANDLE fd, size_t *size)
 	return MDB_SUCCESS;
 }
 
-#ifdef __linux
+#ifdef BROKEN_FDATASYNC
 #include <sys/utsname.h>
 #include <sys/vfs.h>
 #endif
@@ -3952,7 +3958,7 @@ mdb_env_open2(MDB_env *env)
 		env->me_pidquery = PROCESS_QUERY_INFORMATION;
 #endif /* _WIN32 */
 
-#ifdef __linux
+#ifdef BROKEN_FDATASYNC
 	/* ext3/ext4 fdatasync is broken on some older Linux kernels.
 	 * https://lkml.org/lkml/2012/9/3/83
 	 * Kernels after 3.6-rc6 are known good.

From 8b6c4250b108f6f084358ef45cb3a13b7bbd71d2 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 12 Jan 2015 10:38:14 +0000
Subject: [PATCH 045/504] More cleanup for fdatasync hack

---
 libraries/liblmdb/mdb.c | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index d82c3f022b..84cd8a35a0 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -81,7 +81,9 @@ extern int cacheflush(char *addr, int nbytes, int cache);
 
 #if defined(__linux) && !defined(MDB_FDATASYNC_WORKS)
 /** fdatasync is broken on ext3/ext4fs on older kernels, see
- *	description in #mdb_env_open2 comments
+ *	description in #mdb_env_open2 comments. You can safely
+ *	define MDB_FDATASYNC_WORKS if this code will only be run
+ *	on kernels 3.6 and newer.
  */
 #define	BROKEN_FDATASYNC
 #endif
@@ -1144,6 +1146,8 @@ struct MDB_env {
 #define	MDB_ENV_ACTIVE	0x20000000U
 	/** me_txkey is set */
 #define	MDB_ENV_TXKEY	0x10000000U
+	/** fdatasync is unreliable */
+#define	MDB_FSYNCONLY	0x08000000U
 	uint32_t 	me_flags;		/**< @ref mdb_env */
 	unsigned int	me_psize;	/**< DB page size, inited from me_os_psize */
 	unsigned int	me_os_psize;	/**< OS page size, from #GET_PAGESIZE */
@@ -1186,9 +1190,6 @@ struct MDB_env {
 #ifdef _WIN32
 	int		me_pidquery;		/**< Used in OpenProcess */
 #endif
-#ifdef BROKEN_FDATASYNC
-	int		me_fsynconly;		/**< fdatasync is unreliable */
-#endif
 #if defined(_WIN32) || defined(MDB_USE_SYSV_SEM)
 	/* Windows mutexes/SysV semaphores do not reside in shared mem */
 	mdb_mutex_t	me_rmutex;
@@ -2368,7 +2369,7 @@ mdb_env_sync(MDB_env *env, int force)
 #endif
 		} else {
 #ifdef BROKEN_FDATASYNC
-			if (env->me_fsynconly) {
+			if (env->me_flags & MDB_FSYNCONLY) {
 				if (fsync(env->me_fd))
 					rc = ErrCode();
 			} else
@@ -4000,7 +4001,7 @@ mdb_env_open2(MDB_env *env)
 			} else {	/* 4.x and newer is OK */
 				break;
 			}
-			env->me_fsynconly = 1;
+			env->me_flags |= MDB_FSYNCONLY;
 			break;
 		}
 	}

From 9441012435e8417a96e11350a4e74186db5a504a Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Mon, 12 Jan 2015 12:19:51 +0100
Subject: [PATCH 046/504] ITS#8021 doc: Don't mix MDB_WRITEMAP + non-WRITEMAP

---
 libraries/liblmdb/lmdb.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index d22009bbcc..9efdfb3963 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -521,8 +521,8 @@ int  mdb_env_create(MDB_env **env);
 	 *		and uses fewer mallocs, but loses protection from application bugs
 	 *		like wild pointer writes and other bad updates into the database.
 	 *		Incompatible with nested transactions.
-	 *		Processes with and without MDB_WRITEMAP on the same environment do
-	 *		not cooperate well.
+	 *		Do not mix processes with and without MDB_WRITEMAP on the same
+	 *		environment.  This can defeat durability (#mdb_env_sync etc).
 	 *	<li>#MDB_NOMETASYNC
 	 *		Flush system buffers to disk only once per transaction, omit the
 	 *		metadata flush. Defer that until the system flushes files to disk,

From 631970e485a37d8fc270d8de41ecfe76d8521b70 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Mon, 12 Jan 2015 21:02:29 +0100
Subject: [PATCH 047/504] Make SysV semaphores robust. Cleanup MDB_ROBUST.

---
 libraries/liblmdb/lmdb.h |   4 +-
 libraries/liblmdb/mdb.c  | 108 +++++++++++++++++++++++----------------
 2 files changed, 66 insertions(+), 46 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 9efdfb3963..f37d8f5897 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -109,7 +109,9 @@
  *	  The transaction becomes "long-lived" as above until a check
  *	  for stale readers is performed or the lockfile is reset,
  *	  since the process may not remove it from the lockfile.
- *	  Except write-transactions on Unix with MDB_ROBUST or on Windows.
+ *
+ *	  This does not apply to write transactions if the system clears
+ *	  stale writers, see above.
  *
  *	- If you do that anyway, do a periodic check for stale readers. Or
  *	  close the environment once in a while, so the lockfile can get reset.
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 84cd8a35a0..f0fe6154b2 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -208,7 +208,7 @@ union semun {
 #define MDB_DEVEL 0
 #endif
 
-#if defined(_WIN32) || (defined(EOWNERDEAD) && !defined(MDB_USE_SYSV_SEM))
+#if defined(_WIN32) || defined(MDB_USE_SYSV_SEM) || defined(EOWNERDEAD)
 #define MDB_ROBUST_SUPPORTED	1
 #endif
 
@@ -222,6 +222,16 @@ union semun {
 # define mdb_func_	"<mdb_unknown>"
 #endif
 
+/* Internal error codes, not exposed outside liblmdb */
+#define	MDB_NO_ROOT		(MDB_LAST_ERRCODE + 10)
+#ifdef _WIN32
+#define MDB_OWNERDEAD	((int) WAIT_ABANDONED)
+#elif defined MDB_USE_SYSV_SEM
+#define MDB_OWNERDEAD	(MDB_LAST_ERRCODE + 11)
+#else
+#define MDB_OWNERDEAD	EOWNERDEAD
+#endif
+
 #ifdef _WIN32
 #define MDB_USE_HASH	1
 #define MDB_PIDLOCK	0
@@ -237,7 +247,6 @@ typedef HANDLE mdb_mutex_t;
 #define pthread_key_delete(x)	TlsFree(x)
 #define pthread_getspecific(x)	TlsGetValue(x)
 #define pthread_setspecific(x,y)	(TlsSetValue(x,y) ? 0 : ErrCode())
-#define pthread_mutex_consistent(mutex) 0
 #define pthread_mutex_unlock(x)	ReleaseMutex(*x)
 #define pthread_mutex_lock(x)	WaitForSingleObject(*x, INFINITE)
 #define pthread_cond_signal(x)	SetEvent(*x)
@@ -247,6 +256,7 @@ typedef HANDLE mdb_mutex_t;
 #define MDB_MUTEX(env, rw)		((env)->me_##rw##mutex)
 #define LOCK_MUTEX0(mutex)		WaitForSingleObject(mutex, INFINITE)
 #define UNLOCK_MUTEX(mutex)		ReleaseMutex(mutex)
+#define mdb_mutex_consistent(mutex)	0
 #define getpid()	GetCurrentProcessId()
 #define	MDB_FDATASYNC(fd)	(!FlushFileBuffers(fd))
 #define	MDB_MSYNC(addr,len,flags)	(!FlushViewOfFile(addr,len))
@@ -274,6 +284,7 @@ typedef HANDLE mdb_mutex_t;
 typedef struct mdb_mutex {
 	int semid;
 	int semnum;
+	int *locked;
 } mdb_mutex_t;
 
 #define MDB_MUTEX(env, rw)		(&(env)->me_##rw##mutex)
@@ -281,19 +292,28 @@ typedef struct mdb_mutex {
 #define UNLOCK_MUTEX(mutex)		do { \
 	struct sembuf sb = { 0, 1, SEM_UNDO }; \
 	sb.sem_num = (mutex)->semnum; \
+	*(mutex)->locked = 0; \
 	semop((mutex)->semid, &sb, 1); \
 } while(0)
 
 static int
 mdb_sem_wait(mdb_mutex_t *sem)
 {
-   int rc;
-   struct sembuf sb = { 0, -1, SEM_UNDO };
-   sb.sem_num = sem->semnum;
-   while ((rc = semop(sem->semid, &sb, 1)) && (rc = errno) == EINTR) ;
-   return rc;
+	int rc, *locked = sem->locked;
+	struct sembuf sb = { 0, -1, SEM_UNDO };
+	sb.sem_num = sem->semnum;
+	do {
+		if (!semop(sem->semid, &sb, 1)) {
+			rc = *locked ? MDB_OWNERDEAD : MDB_SUCCESS;
+			*locked = 1;
+			break;
+		}
+	} while ((rc = errno) == EINTR);
+	return rc;
 }
 
+#define mdb_mutex_consistent(mutex)	0
+
 #else
 	/** Pointer/HANDLE type of shared mutex/semaphore.
 	 */
@@ -308,6 +328,9 @@ typedef pthread_mutex_t mdb_mutex_t;
 	/** Unlock the reader or writer mutex.
 	 */
 #define UNLOCK_MUTEX(mutex)	pthread_mutex_unlock(mutex)
+	/** Mark mutex-protected data as repaired, after death of previous owner.
+	 */
+#define mdb_mutex_consistent(mutex)	pthread_mutex_consistent(mutex)
 #endif	/* MDB_USE_SYSV_SEM */
 
 	/** Get the error code for the last failed system function.
@@ -336,11 +359,17 @@ typedef pthread_mutex_t mdb_mutex_t;
 #if defined(_WIN32)
 #define MNAME_LEN	32
 #elif defined(MDB_USE_SYSV_SEM)
-#define MNAME_LEN	0
+#define MNAME_LEN	(sizeof(int))
 #else
 #define MNAME_LEN	(sizeof(pthread_mutex_t))
 #endif
 
+#ifdef MDB_USE_SYSV_SEM
+#define SYSV_SEM_FLAG	1		/**< SysV sems in lockfile format */
+#else
+#define SYSV_SEM_FLAG	0
+#endif
+
 /** @} */
 
 #ifdef MDB_ROBUST_SUPPORTED
@@ -667,6 +696,7 @@ typedef struct MDB_txbody {
 	char	mtb_rmname[MNAME_LEN];
 #elif defined(MDB_USE_SYSV_SEM)
 	int 	mtb_semid;
+	int		mtb_rlocked;
 #else
 		/** Mutex protecting access to this table.
 		 *	This is the #MDB_MUTEX(env,r) reader table lock.
@@ -695,22 +725,25 @@ typedef struct MDB_txninfo {
 #define mti_rmname	mt1.mtb.mtb_rmname
 #define mti_txnid	mt1.mtb.mtb_txnid
 #define mti_numreaders	mt1.mtb.mtb_numreaders
-		char pad[(sizeof(MDB_txbody)+CACHELINE-1) & ~(CACHELINE-1)];
-	} mt1;
 #ifdef MDB_USE_SYSV_SEM
 #define	mti_semid	mt1.mtb.mtb_semid
-#else
+#define	mti_rlocked	mt1.mtb.mtb_rlocked
+#endif
+		char pad[(sizeof(MDB_txbody)+CACHELINE-1) & ~(CACHELINE-1)];
+	} mt1;
 	union {
 #if defined(_WIN32)
 		char mt2_wmname[MNAME_LEN];
 #define	mti_wmname	mt2.mt2_wmname
+#elif defined MDB_USE_SYSV_SEM
+		int mt2_wlocked;
+#define mti_wlocked	mt2.mt2_wlocked
 #else
 		pthread_mutex_t	mt2_wmutex;
 #define mti_wmutex	mt2.mt2_wmutex
 #endif
 		char pad[(MNAME_LEN+CACHELINE-1) & ~(CACHELINE-1)];
 	} mt2;
-#endif
 	MDB_reader	mti_readers[1];
 } MDB_txninfo;
 
@@ -719,7 +752,7 @@ typedef struct MDB_txninfo {
 	((uint32_t) \
 	 ((MDB_LOCK_VERSION) \
 	  /* Flags which describe functionality */ \
-	  + (((MNAME_LEN) == 0)   << 18) /* MDB_USE_SYSV_SEM */ \
+	  + (SYSV_SEM_FLAG << 18) \
 	  + (((MDB_PIDLOCK) != 0) << 16)))
 /** @} */
 
@@ -2596,17 +2629,8 @@ mdb_txn_renew0(MDB_txn *txn)
 		if (ti) {
 			if (LOCK_MUTEX(rc, env, MDB_MUTEX(env, w)))
 				return rc;
-#ifdef MDB_USE_SYSV_SEM
-			meta = env->me_metas[ mdb_env_pick_meta(env) ];
-			txn->mt_txnid = meta->mm_txnid;
-			/* Update mti_txnid like mdb_mutex_failed() would,
-			 * in case last writer crashed before updating it.
-			 */
-			ti->mti_txnid = txn->mt_txnid;
-#else
 			txn->mt_txnid = ti->mti_txnid;
 			meta = env->me_metas[txn->mt_txnid & 1];
-#endif
 		} else {
 			meta = env->me_metas[ mdb_env_pick_meta(env) ];
 			txn->mt_txnid = meta->mm_txnid;
@@ -4358,6 +4382,10 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 	int fdflags;
 #	define MDB_CLOEXEC		0
 #endif
+#endif
+#ifdef MDB_USE_SYSV_SEM
+	int semid;
+	union semun semu;
 #endif
 	int rc;
 	off_t size, rsize;
@@ -4472,17 +4500,10 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 		env->me_wmutex = CreateMutex(&mdb_all_sa, FALSE, env->me_txns->mti_wmname);
 		if (!env->me_wmutex) goto fail_errno;
 #elif defined(MDB_USE_SYSV_SEM)
-		union semun semu;
 		unsigned short vals[2] = {1, 1};
-		int semid = semget(IPC_PRIVATE, 2, mode);
+		semid = semget(IPC_PRIVATE, 2, mode);
 		if (semid < 0)
 			goto fail_errno;
-
-		env->me_rmutex.semid = semid;
-		env->me_wmutex.semid = semid;
-		env->me_rmutex.semnum = 0;
-		env->me_wmutex.semnum = 1;
-
 		semu.array = vals;
 		if (semctl(semid, 0, SETALL, semu) < 0)
 			goto fail_errno;
@@ -4509,8 +4530,6 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 	} else {
 #ifdef MDB_USE_SYSV_SEM
 		struct semid_ds buf;
-		union semun semu;
-		int semid;
 #endif
 		if (env->me_txns->mti_magic != MDB_MAGIC) {
 			DPUTS("lock region has invalid magic");
@@ -4535,20 +4554,23 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 #elif defined(MDB_USE_SYSV_SEM)
 		semid = env->me_txns->mti_semid;
 		semu.buf = &buf;
-
 		/* check for read access */
 		if (semctl(semid, 0, IPC_STAT, semu) < 0)
 			goto fail_errno;
 		/* check for write access */
 		if (semctl(semid, 0, IPC_SET, semu) < 0)
 			goto fail_errno;
-
-		env->me_rmutex.semid = semid;
-		env->me_wmutex.semid = semid;
-		env->me_rmutex.semnum = 0;
-		env->me_wmutex.semnum = 1;
 #endif
 	}
+#ifdef MDB_USE_SYSV_SEM
+	env->me_rmutex.semid = semid;
+	env->me_wmutex.semid = semid;
+	env->me_rmutex.semnum = 0;
+	env->me_wmutex.semnum = 1;
+	env->me_rmutex.locked = &env->me_txns->mti_rlocked;
+	env->me_wmutex.locked = &env->me_txns->mti_wlocked;
+#endif
+
 	return MDB_SUCCESS;
 
 fail_errno:
@@ -6140,7 +6162,6 @@ int
 mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data,
     unsigned int flags)
 {
-	enum { MDB_NO_ROOT = MDB_LAST_ERRCODE+10 }; /* internal code */
 	MDB_env		*env;
 	MDB_node	*leaf = NULL;
 	MDB_page	*fp, *mp;
@@ -9593,7 +9614,7 @@ static int mdb_reader_check0(MDB_env *env, int rlocked, int *dead)
 
 #ifdef MDB_ROBUST_SUPPORTED
 /** Handle #LOCK_MUTEX0() failure.
- * With #MDB_ROBUST, try to repair the lock file if the mutex owner died.
+ * Try to repair the lock file if the mutex owner died.
  * @param[in] env	the environment handle
  * @param[in] mutex	LOCK_MUTEX0() mutex
  * @param[in] rc	LOCK_MUTEX0() error (nonzero)
@@ -9602,11 +9623,8 @@ static int mdb_reader_check0(MDB_env *env, int rlocked, int *dead)
 static int mdb_mutex_failed(MDB_env *env, mdb_mutex_t *mutex, int rc)
 {
 	int toggle, rlocked, rc2;
-#ifndef _WIN32
-	enum { WAIT_ABANDONED = EOWNERDEAD };
-#endif
 
-	if (rc == (int) WAIT_ABANDONED) {
+	if (rc == MDB_OWNERDEAD) {
 		/* We own the mutex. Clean up after dead previous owner. */
 		rc = MDB_SUCCESS;
 		rlocked = (mutex == MDB_MUTEX(env, r));
@@ -9627,7 +9645,7 @@ static int mdb_mutex_failed(MDB_env *env, mdb_mutex_t *mutex, int rc)
 			(rc ? "this process' env is hosed" : "recovering")));
 		rc2 = mdb_reader_check0(env, rlocked, NULL);
 		if (rc2 == 0)
-			rc2 = pthread_mutex_consistent(mutex);
+			rc2 = mdb_mutex_consistent(mutex);
 		if (rc || (rc = rc2)) {
 			DPRINTF(("LOCK_MUTEX recovery failed, %s", mdb_strerror(rc)));
 			UNLOCK_MUTEX(mutex);

From d711e357da8432f4e5d32ea5a90b36a711e8716e Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Mon, 12 Jan 2015 21:02:29 +0100
Subject: [PATCH 048/504] Whitespace (align with mdb.RE)

---
 libraries/liblmdb/lmdb.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index f37d8f5897..ebfbc5dc82 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -450,7 +450,7 @@ typedef struct MDB_envinfo {
 	void	*me_mapaddr;			/**< Address of map, if fixed */
 	size_t	me_mapsize;				/**< Size of the data memory map */
 	size_t	me_last_pgno;			/**< ID of the last used page */
-	size_t  me_last_txnid;		/**< ID of the last committed transaction */
+	size_t	me_last_txnid;			/**< ID of the last committed transaction */
 	unsigned int me_maxreaders;		/**< max reader slots in the environment */
 	unsigned int me_numreaders;		/**< max reader slots used in the environment */
 } MDB_envinfo;

From 71741a6b08b18e5be4e57c211e70541f4e4b4995 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 13 Jan 2015 18:47:18 +0000
Subject: [PATCH 049/504] Fix potential null deref (coverity)

---
 libraries/liblmdb/mdb.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index f0fe6154b2..c4dc269037 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -4752,13 +4752,15 @@ mdb_env_close0(MDB_env *env, int excl)
 		return;
 
 	/* Doing this here since me_dbxs may not exist during mdb_env_close */
-	for (i = env->me_maxdbs; --i > MAIN_DBI; )
-		free(env->me_dbxs[i].md_name.mv_data);
+	if (env->me_dbxs) {
+		for (i = env->me_maxdbs; --i > MAIN_DBI; )
+			free(env->me_dbxs[i].md_name.mv_data);
+		free(env->me_dbxs);
+	}
 
 	free(env->me_pbuf);
 	free(env->me_dbiseqs);
 	free(env->me_dbflags);
-	free(env->me_dbxs);
 	free(env->me_path);
 	free(env->me_dirty_list);
 	free(env->me_txn0);
@@ -7554,7 +7556,7 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst)
 			cdst->mc_ki[cdst->mc_top] = 0;
 			rc = mdb_update_key(cdst, &nullkey);
 			cdst->mc_ki[cdst->mc_top] = ix;
-			mdb_cassert(csrc, rc == MDB_SUCCESS);
+			mdb_cassert(cdst, rc == MDB_SUCCESS);
 		}
 	}
 

From 404697b369bf6eacab02f68113eeafaaaef85d27 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Wed, 14 Jan 2015 08:12:50 +0100
Subject: [PATCH 050/504] mdb_env_set_flags(): Reject CHANGELESS flags.

Reverts part of a2ac10107e2fb845c4a38a339239063ec4407d84.
MDB_ROBUST needed to be accepted, but that flag is gone.
---
 libraries/liblmdb/lmdb.h | 1 -
 libraries/liblmdb/mdb.c  | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index ebfbc5dc82..e95fff344b 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -736,7 +736,6 @@ void mdb_env_close(MDB_env *env);
 	 * This may be used to set some flags in addition to those from
 	 * #mdb_env_open(), or to unset these flags.  If several threads
 	 * change the flags at the same time, the result is undefined.
-	 * Most flags cannot be changed after #mdb_env_open().
 	 * @param[in] env An environment handle returned by #mdb_env_create()
 	 * @param[in] flags The flags to change, bitwise OR'ed together
 	 * @param[in] onoff A non-zero value sets the flags, zero clears them.
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index c4dc269037..e2aa170357 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -9001,7 +9001,7 @@ mdb_env_copy(MDB_env *env, const char *path)
 int ESECT
 mdb_env_set_flags(MDB_env *env, unsigned int flag, int onoff)
 {
-	if (flag & (env->me_map ? ~CHANGEABLE : ~(CHANGEABLE|CHANGELESS)))
+	if ((flag & CHANGEABLE) != flag)
 		return EINVAL;
 	if (onoff)
 		env->me_flags |= flag;

From b2ab9910dd5f13ca2d92eeb4c39a90b922df8dfe Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 17 Jan 2015 06:32:12 +0100
Subject: [PATCH 051/504] ITS#7971 Fix reader allocation and me_numreaders

---
 libraries/liblmdb/mdb.c | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index e2aa170357..7175276ab0 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1185,7 +1185,8 @@ struct MDB_env {
 	unsigned int	me_psize;	/**< DB page size, inited from me_os_psize */
 	unsigned int	me_os_psize;	/**< OS page size, from #GET_PAGESIZE */
 	unsigned int	me_maxreaders;	/**< size of the reader table */
-	unsigned int	me_numreaders;	/**< max numreaders set by this env */
+	/** Max #MDB_txninfo.%mti_numreaders of interest to #mdb_env_close() */
+	volatile int	me_close_readers;
 	MDB_dbi		me_numdbs;		/**< number of DBs opened */
 	MDB_dbi		me_maxdbs;		/**< size of the DB table */
 	MDB_PID_T	me_pid;		/**< process ID of this env */
@@ -2603,13 +2604,19 @@ mdb_txn_renew0(MDB_txn *txn)
 					return MDB_READERS_FULL;
 				}
 				r = &ti->mti_readers[i];
+				/* Claim the reader slot, carefully since other code
+				 * uses the reader table un-mutexed: First reset the
+				 * slot, next publish it in mti_numreaders.  After
+				 * that, it is safe for mdb_env_close() to touch it.
+				 * When it will be closed, we can finally claim it.
+				 */
+				r->mr_pid = 0;
 				r->mr_txnid = (txnid_t)-1;
 				r->mr_tid = tid;
-				r->mr_pid = pid; /* should be written last, see ITS#7971. */
 				if (i == nr)
 					ti->mti_numreaders = ++nr;
-				/* Save numreaders for un-mutexed mdb_env_close() */
-				env->me_numreaders = nr;
+				env->me_close_readers = nr;
+				r->mr_pid = pid;
 				UNLOCK_MUTEX(rmutex);
 
 				new_notls = (env->me_flags & MDB_NOTLS);
@@ -4790,8 +4797,12 @@ mdb_env_close0(MDB_env *env, int excl)
 		MDB_PID_T pid = env->me_pid;
 		/* Clearing readers is done in this function because
 		 * me_txkey with its destructor must be disabled first.
+		 *
+		 * We skip the the reader mutex, so we touch only
+		 * data owned by this process (me_close_readers and
+		 * our readers), and clear each reader atomically.
 		 */
-		for (i = env->me_numreaders; --i >= 0; )
+		for (i = env->me_close_readers; --i >= 0; )
 			if (env->me_txns->mti_readers[i].mr_pid == pid)
 				env->me_txns->mti_readers[i].mr_pid = 0;
 #ifdef _WIN32
@@ -9110,11 +9121,7 @@ mdb_env_info(MDB_env *env, MDB_envinfo *arg)
 	arg->me_mapaddr = env->me_metas[toggle]->mm_address;
 	arg->me_mapsize = env->me_mapsize;
 	arg->me_maxreaders = env->me_maxreaders;
-
-	/* me_numreaders may be zero if this process never used any readers. Use
-	 * the shared numreader count if it exists.
-	 */
-	arg->me_numreaders = env->me_txns ? env->me_txns->mti_numreaders : env->me_numreaders;
+	arg->me_numreaders = env->me_txns ? env->me_txns->mti_numreaders : 0;
 
 	arg->me_last_pgno = env->me_metas[toggle]->mm_last_pg;
 	arg->me_last_txnid = env->me_metas[toggle]->mm_txnid;

From 570ba6fb86d2dbf7b1a86d2f01aba4c885ca4b45 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 27 Jan 2015 06:10:57 +0100
Subject: [PATCH 052/504] Fix EACCES description

---
 libraries/liblmdb/lmdb.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index e95fff344b..4f857201b5 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -1468,7 +1468,7 @@ int  mdb_cursor_get(MDB_cursor *cursor, MDB_val *key, MDB_val *data,
 	 * <ul>
 	 *	<li>#MDB_MAP_FULL - the database is full, see #mdb_env_set_mapsize().
 	 *	<li>#MDB_TXN_FULL - the transaction has too many dirty pages.
-	 *	<li>EACCES - an attempt was made to modify a read-only database.
+	 *	<li>EACCES - an attempt was made to write in a read-only transaction.
 	 *	<li>EINVAL - an invalid parameter was specified.
 	 * </ul>
 	 */
@@ -1488,7 +1488,7 @@ int  mdb_cursor_put(MDB_cursor *cursor, MDB_val *key, MDB_val *data,
 	 * @return A non-zero error value on failure and 0 on success. Some possible
 	 * errors are:
 	 * <ul>
-	 *	<li>EACCES - an attempt was made to modify a read-only database.
+	 *	<li>EACCES - an attempt was made to write in a read-only transaction.
 	 *	<li>EINVAL - an invalid parameter was specified.
 	 * </ul>
 	 */

From 3368d1f5e243225cba4d730fba19ff600798ebe3 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Fri, 30 Jan 2015 09:21:05 +0000
Subject: [PATCH 053/504] Support MDB_NOSYNC on mdb_txn_begin()

---
 libraries/liblmdb/lmdb.h |  2 ++
 libraries/liblmdb/mdb.c  | 17 +++++++++++++----
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 4f857201b5..61318ffd60 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -930,6 +930,8 @@ int  mdb_env_set_assert(MDB_env *env, MDB_assert_func *func);
 	 * <ul>
 	 *	<li>#MDB_RDONLY
 	 *		This transaction will not perform any write operations.
+	 *	<li>#MDB_NOSYNC
+	 *		Don't flush system buffers to disk when committing this transaction.
 	 * </ul>
 	 * @param[out] txn Address where the new #MDB_txn handle will be stored
 	 * @return A non-zero error value on failure and 0 on success. Some possible
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 7175276ab0..984a5bdd16 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1084,6 +1084,7 @@ struct MDB_txn {
 #define MDB_TXN_ERROR		0x02		/**< txn is unusable after an error */
 #define MDB_TXN_DIRTY		0x04		/**< must write, even if dirty list is empty */
 #define MDB_TXN_SPILLS		0x08		/**< txn or a parent has spilled pages */
+#define MDB_TXN_NOSYNC		0x10		/**< don't sync this txn on commit */
 /** @} */
 	unsigned int	mt_flags;		/**< @ref mdb_txn */
 	/** #dirty_list room: Array size - \#dirty pages visible to this txn.
@@ -2755,6 +2756,8 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 		txn->mt_dbflags = (unsigned char *)(txn->mt_dbs + env->me_maxdbs);
 		txn->mt_dbiseqs = env->me_dbiseqs;
 	} else {
+		if (flags & MDB_NOSYNC)
+			txn->mt_flags |= MDB_TXN_NOSYNC;
 		txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs);
 		if (parent) {
 			txn->mt_dbiseqs = parent->mt_dbiseqs;
@@ -3486,9 +3489,12 @@ mdb_txn_commit(MDB_txn *txn)
 	mdb_audit(txn);
 #endif
 
-	if ((rc = mdb_page_flush(txn, 0)) ||
-		(rc = mdb_env_sync(env, 0)) ||
-		(rc = mdb_env_write_meta(txn)))
+	if ((rc = mdb_page_flush(txn, 0)))
+		goto fail;
+	if (!F_ISSET(txn->mt_flags, MDB_TXN_NOSYNC) &&
+		(rc = mdb_env_sync(env, 0)))
+		goto fail;
+	if ((rc = mdb_env_write_meta(txn)))
 		goto fail;
 
 	/* Free P_LOOSE pages left behind in dirty_list */
@@ -3682,6 +3688,8 @@ mdb_env_write_meta(MDB_txn *txn)
 		__sync_synchronize();
 #endif
 		mp->mm_txnid = txn->mt_txnid;
+		if (F_ISSET(txn->mt_flags, MDB_TXN_NOSYNC))
+			goto done;
 		if (!(env->me_flags & (MDB_NOMETASYNC|MDB_NOSYNC))) {
 			unsigned meta_size = env->me_psize;
 			rc = (env->me_flags & MDB_MAPASYNC) ? MS_ASYNC : MS_SYNC;
@@ -3718,7 +3726,8 @@ mdb_env_write_meta(MDB_txn *txn)
 	off += PAGEHDRSZ;
 
 	/* Write to the SYNC fd */
-	mfd = env->me_flags & (MDB_NOSYNC|MDB_NOMETASYNC) ?
+	mfd = ((env->me_flags & (MDB_NOSYNC|MDB_NOMETASYNC)) ||
+		(txn->mt_flags & MDB_TXN_NOSYNC)) ?
 		env->me_fd : env->me_mfd;
 #ifdef _WIN32
 	{

From 8bb541bc408c6f136558a490dacd7abd3c339cec Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 2 Feb 2015 10:33:41 +0000
Subject: [PATCH 054/504] Also support NOMETASYNC on txn_begin

---
 libraries/liblmdb/lmdb.h | 2 ++
 libraries/liblmdb/mdb.c  | 7 +++++--
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 61318ffd60..d5c70dad94 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -932,6 +932,8 @@ int  mdb_env_set_assert(MDB_env *env, MDB_assert_func *func);
 	 *		This transaction will not perform any write operations.
 	 *	<li>#MDB_NOSYNC
 	 *		Don't flush system buffers to disk when committing this transaction.
+	 *	<li>#MDB_NOMETASYNC
+	 *		Flush system buffers but omit metadata flush when committing this transaction.
 	 * </ul>
 	 * @param[out] txn Address where the new #MDB_txn handle will be stored
 	 * @return A non-zero error value on failure and 0 on success. Some possible
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 984a5bdd16..ffe0ad45ea 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1085,6 +1085,7 @@ struct MDB_txn {
 #define MDB_TXN_DIRTY		0x04		/**< must write, even if dirty list is empty */
 #define MDB_TXN_SPILLS		0x08		/**< txn or a parent has spilled pages */
 #define MDB_TXN_NOSYNC		0x10		/**< don't sync this txn on commit */
+#define MDB_TXN_NOMETASYNC	0x20		/**< don't sync meta for this txn on commit */
 /** @} */
 	unsigned int	mt_flags;		/**< @ref mdb_txn */
 	/** #dirty_list room: Array size - \#dirty pages visible to this txn.
@@ -2758,6 +2759,8 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 	} else {
 		if (flags & MDB_NOSYNC)
 			txn->mt_flags |= MDB_TXN_NOSYNC;
+		if (flags & MDB_NOMETASYNC)
+			txn->mt_flags |= MDB_TXN_NOMETASYNC;
 		txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs);
 		if (parent) {
 			txn->mt_dbiseqs = parent->mt_dbiseqs;
@@ -3688,7 +3691,7 @@ mdb_env_write_meta(MDB_txn *txn)
 		__sync_synchronize();
 #endif
 		mp->mm_txnid = txn->mt_txnid;
-		if (F_ISSET(txn->mt_flags, MDB_TXN_NOSYNC))
+		if (txn->mt_flags & (MDB_TXN_NOSYNC|MDB_TXN_NOMETASYNC))
 			goto done;
 		if (!(env->me_flags & (MDB_NOMETASYNC|MDB_NOSYNC))) {
 			unsigned meta_size = env->me_psize;
@@ -3727,7 +3730,7 @@ mdb_env_write_meta(MDB_txn *txn)
 
 	/* Write to the SYNC fd */
 	mfd = ((env->me_flags & (MDB_NOSYNC|MDB_NOMETASYNC)) ||
-		(txn->mt_flags & MDB_TXN_NOSYNC)) ?
+		(txn->mt_flags & (MDB_TXN_NOSYNC|MDB_TXN_NOMETASYNC))) ?
 		env->me_fd : env->me_mfd;
 #ifdef _WIN32
 	{

From f526f1b9a33bf50e5cda858e6130524d4fdfe37a Mon Sep 17 00:00:00 2001
From: Quanah Gibson-Mount <quanah@openldap.org>
Date: Thu, 12 Feb 2015 10:46:33 -0600
Subject: [PATCH 055/504] Happy New Year

---
 libraries/liblmdb/midl.c | 2 +-
 libraries/liblmdb/midl.h | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/midl.c b/libraries/liblmdb/midl.c
index 88a3aff10c..16782dcafc 100644
--- a/libraries/liblmdb/midl.c
+++ b/libraries/liblmdb/midl.c
@@ -3,7 +3,7 @@
 /* $OpenLDAP$ */
 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
  *
- * Copyright 2000-2014 The OpenLDAP Foundation.
+ * Copyright 2000-2015 The OpenLDAP Foundation.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/midl.h b/libraries/liblmdb/midl.h
index a7f25026ce..9b041d72d1 100644
--- a/libraries/liblmdb/midl.h
+++ b/libraries/liblmdb/midl.h
@@ -11,7 +11,7 @@
 /* $OpenLDAP$ */
 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
  *
- * Copyright 2000-2014 The OpenLDAP Foundation.
+ * Copyright 2000-2015 The OpenLDAP Foundation.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without

From 6fe12f3273f8a28cd7183e176c5d76f214f77c98 Mon Sep 17 00:00:00 2001
From: Quanah Gibson-Mount <quanah@openldap.org>
Date: Thu, 12 Feb 2015 11:17:59 -0600
Subject: [PATCH 056/504] Happy New Year

---
 libraries/liblmdb/COPYRIGHT      | 2 +-
 libraries/liblmdb/lmdb.h         | 2 +-
 libraries/liblmdb/mdb.c          | 2 +-
 libraries/liblmdb/mdb_copy.1     | 2 +-
 libraries/liblmdb/mdb_copy.c     | 2 +-
 libraries/liblmdb/mdb_dump.1     | 2 +-
 libraries/liblmdb/mdb_dump.c     | 2 +-
 libraries/liblmdb/mdb_load.1     | 2 +-
 libraries/liblmdb/mdb_load.c     | 2 +-
 libraries/liblmdb/mdb_stat.1     | 2 +-
 libraries/liblmdb/mdb_stat.c     | 2 +-
 libraries/liblmdb/mtest.c        | 2 +-
 libraries/liblmdb/mtest2.c       | 2 +-
 libraries/liblmdb/mtest3.c       | 2 +-
 libraries/liblmdb/mtest4.c       | 2 +-
 libraries/liblmdb/mtest5.c       | 2 +-
 libraries/liblmdb/mtest6.c       | 2 +-
 libraries/liblmdb/sample-bdb.txt | 2 +-
 libraries/liblmdb/sample-mdb.txt | 2 +-
 19 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/libraries/liblmdb/COPYRIGHT b/libraries/liblmdb/COPYRIGHT
index e0eb484c7a..722d1a5154 100644
--- a/libraries/liblmdb/COPYRIGHT
+++ b/libraries/liblmdb/COPYRIGHT
@@ -1,4 +1,4 @@
-Copyright 2011-2014 Howard Chu, Symas Corp.
+Copyright 2011-2015 Howard Chu, Symas Corp.
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index d5c70dad94..736d19033b 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -126,7 +126,7 @@
  *
  *	@author	Howard Chu, Symas Corporation.
  *
- *	@copyright Copyright 2011-2014 Howard Chu, Symas Corp. All rights reserved.
+ *	@copyright Copyright 2011-2015 Howard Chu, Symas Corp. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted only as authorized by the OpenLDAP
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index ffe0ad45ea..6d6b6cfe77 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5,7 +5,7 @@
  *	BerkeleyDB API, but much simplified.
  */
 /*
- * Copyright 2011-2014 Howard Chu, Symas Corp.
+ * Copyright 2011-2015 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mdb_copy.1 b/libraries/liblmdb/mdb_copy.1
index 094b260563..1e2a97694f 100644
--- a/libraries/liblmdb/mdb_copy.1
+++ b/libraries/liblmdb/mdb_copy.1
@@ -1,5 +1,5 @@
 .TH MDB_COPY 1 "2014/06/20" "LMDB 0.9.14"
-.\" Copyright 2012-2014 Howard Chu, Symas Corp. All Rights Reserved.
+.\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_copy \- LMDB environment copy tool
diff --git a/libraries/liblmdb/mdb_copy.c b/libraries/liblmdb/mdb_copy.c
index af0a941110..f37ccbcc21 100644
--- a/libraries/liblmdb/mdb_copy.c
+++ b/libraries/liblmdb/mdb_copy.c
@@ -1,6 +1,6 @@
 /* mdb_copy.c - memory-mapped database backup tool */
 /*
- * Copyright 2012 Howard Chu, Symas Corp.
+ * Copyright 2012-2015 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mdb_dump.1 b/libraries/liblmdb/mdb_dump.1
index 6fcc930811..dd545049b2 100644
--- a/libraries/liblmdb/mdb_dump.1
+++ b/libraries/liblmdb/mdb_dump.1
@@ -1,5 +1,5 @@
 .TH MDB_DUMP 1 "2014/06/20" "LMDB 0.9.14"
-.\" Copyright 2014 Howard Chu, Symas Corp. All Rights Reserved.
+.\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_dump \- LMDB environment export tool
diff --git a/libraries/liblmdb/mdb_dump.c b/libraries/liblmdb/mdb_dump.c
index 0eb85fd20e..7202d865bc 100644
--- a/libraries/liblmdb/mdb_dump.c
+++ b/libraries/liblmdb/mdb_dump.c
@@ -1,6 +1,6 @@
 /* mdb_dump.c - memory-mapped database dump tool */
 /*
- * Copyright 2011-2014 Howard Chu, Symas Corp.
+ * Copyright 2011-2015 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mdb_load.1 b/libraries/liblmdb/mdb_load.1
index 7280240103..511ec552b4 100644
--- a/libraries/liblmdb/mdb_load.1
+++ b/libraries/liblmdb/mdb_load.1
@@ -1,5 +1,5 @@
 .TH MDB_LOAD 1 "2014/06/20" "LMDB 0.9.14"
-.\" Copyright 2014 Howard Chu, Symas Corp. All Rights Reserved.
+.\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_load \- LMDB environment import tool
diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c
index e0d95e13c3..f6266923b5 100644
--- a/libraries/liblmdb/mdb_load.c
+++ b/libraries/liblmdb/mdb_load.c
@@ -1,6 +1,6 @@
 /* mdb_load.c - memory-mapped database load tool */
 /*
- * Copyright 2011-2014 Howard Chu, Symas Corp.
+ * Copyright 2011-2015 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mdb_stat.1 b/libraries/liblmdb/mdb_stat.1
index 3d8d461d99..e6ee5ad585 100644
--- a/libraries/liblmdb/mdb_stat.1
+++ b/libraries/liblmdb/mdb_stat.1
@@ -1,5 +1,5 @@
 .TH MDB_STAT 1 "2014/06/20" "LMDB 0.9.14"
-.\" Copyright 2012-2014 Howard Chu, Symas Corp. All Rights Reserved.
+.\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_stat \- LMDB environment status tool
diff --git a/libraries/liblmdb/mdb_stat.c b/libraries/liblmdb/mdb_stat.c
index 1e9229296f..210609b324 100644
--- a/libraries/liblmdb/mdb_stat.c
+++ b/libraries/liblmdb/mdb_stat.c
@@ -1,6 +1,6 @@
 /* mdb_stat.c - memory-mapped database status tool */
 /*
- * Copyright 2011-2013 Howard Chu, Symas Corp.
+ * Copyright 2011-2015 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest.c b/libraries/liblmdb/mtest.c
index 79b4175e79..66dabc752b 100644
--- a/libraries/liblmdb/mtest.c
+++ b/libraries/liblmdb/mtest.c
@@ -1,6 +1,6 @@
 /* mtest.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2014 Howard Chu, Symas Corp.
+ * Copyright 2011-2015 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest2.c b/libraries/liblmdb/mtest2.c
index f1a3dbd6c8..33e0e741c5 100644
--- a/libraries/liblmdb/mtest2.c
+++ b/libraries/liblmdb/mtest2.c
@@ -1,6 +1,6 @@
 /* mtest2.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2014 Howard Chu, Symas Corp.
+ * Copyright 2011-2015 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest3.c b/libraries/liblmdb/mtest3.c
index f705c52dec..85872c67ed 100644
--- a/libraries/liblmdb/mtest3.c
+++ b/libraries/liblmdb/mtest3.c
@@ -1,6 +1,6 @@
 /* mtest3.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2014 Howard Chu, Symas Corp.
+ * Copyright 2011-2015 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest4.c b/libraries/liblmdb/mtest4.c
index da5a953044..fdcd46a373 100644
--- a/libraries/liblmdb/mtest4.c
+++ b/libraries/liblmdb/mtest4.c
@@ -1,6 +1,6 @@
 /* mtest4.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2014 Howard Chu, Symas Corp.
+ * Copyright 2011-2015 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest5.c b/libraries/liblmdb/mtest5.c
index 39a8c728ac..5295bce41f 100644
--- a/libraries/liblmdb/mtest5.c
+++ b/libraries/liblmdb/mtest5.c
@@ -1,6 +1,6 @@
 /* mtest5.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2014 Howard Chu, Symas Corp.
+ * Copyright 2011-2015 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest6.c b/libraries/liblmdb/mtest6.c
index 07d5758932..b351c1a06a 100644
--- a/libraries/liblmdb/mtest6.c
+++ b/libraries/liblmdb/mtest6.c
@@ -1,6 +1,6 @@
 /* mtest6.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2014 Howard Chu, Symas Corp.
+ * Copyright 2011-2015 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/sample-bdb.txt b/libraries/liblmdb/sample-bdb.txt
index 6a959bd685..563807a2b0 100644
--- a/libraries/liblmdb/sample-bdb.txt
+++ b/libraries/liblmdb/sample-bdb.txt
@@ -3,7 +3,7 @@
  * Do a line-by-line comparison of this and sample-mdb.txt
  */
 /*
- * Copyright 2012 Howard Chu, Symas Corp.
+ * Copyright 2012-2015 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/sample-mdb.txt b/libraries/liblmdb/sample-mdb.txt
index a233ec5dd3..d311f8e954 100644
--- a/libraries/liblmdb/sample-mdb.txt
+++ b/libraries/liblmdb/sample-mdb.txt
@@ -3,7 +3,7 @@
  * Do a line-by-line comparison of this and sample-bdb.txt
  */
 /*
- * Copyright 2012 Howard Chu, Symas Corp.
+ * Copyright 2012-2015 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without

From 04af46b1c1d7c2507aa2a4b9d3f1bc292bc614ee Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 26 Feb 2015 21:36:04 +0000
Subject: [PATCH 057/504] ITS#8066 fix mdb_load with large values

---
 libraries/liblmdb/mdb_load.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c
index f6266923b5..1f6ce0b7ec 100644
--- a/libraries/liblmdb/mdb_load.c
+++ b/libraries/liblmdb/mdb_load.c
@@ -176,7 +176,7 @@ static int unhex(unsigned char *c2)
 static int readline(MDB_val *out, MDB_val *buf)
 {
 	unsigned char *c1, *c2, *end;
-	size_t len;
+	size_t len, l2;
 	int c;
 
 	if (!(mode & NOHDR)) {
@@ -206,6 +206,7 @@ badend:
 
 	c1 = buf->mv_data;
 	len = strlen((char *)c1);
+	l2 = len;
 
 	/* Is buffer too short? */
 	while (c1[len-1] != '\n') {
@@ -217,17 +218,18 @@ badend:
 			return EOF;
 		}
 		c1 = buf->mv_data;
-		c1 += buf->mv_size;
-		if (fgets((char *)c1, buf->mv_size, stdin) == NULL) {
+		c1 += l2;
+		if (fgets((char *)c1, buf->mv_size+1, stdin) == NULL) {
 			Eof = 1;
 			badend();
 			return EOF;
 		}
 		buf->mv_size *= 2;
 		len = strlen((char *)c1);
+		l2 += len;
 	}
 	c1 = c2 = buf->mv_data;
-	len = strlen((char *)c1);
+	len = l2;
 	c1[--len] = '\0';
 	end = c1 + len;
 

From d2dab3c9054abed2c41b65a7b0a658e4458d51ef Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 18 Mar 2015 19:26:21 +0000
Subject: [PATCH 058/504] ITS#8062 fix uninit'd cursor index

---
 libraries/liblmdb/mdb.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 6d6b6cfe77..213842a6e4 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7206,6 +7206,7 @@ mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx)
 	mc->mc_snum = 0;
 	mc->mc_top = 0;
 	mc->mc_pg[0] = 0;
+	mc->mc_ki[0] = 0;
 	mc->mc_flags = 0;
 	if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) {
 		mdb_tassert(txn, mx != NULL);

From 49bb9e82e866bd1b7851836b0b1b23934a7939b3 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Wed, 8 Apr 2015 21:47:24 +0200
Subject: [PATCH 059/504] mdb_txn_renew(): Clear error from previous txn

---
 libraries/liblmdb/mdb.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 213842a6e4..8c69e9fa43 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2570,6 +2570,7 @@ mdb_txn_renew0(MDB_txn *txn)
 	int rc, new_notls = 0;
 
 	if (txn->mt_flags & MDB_TXN_RDONLY) {
+		txn->mt_flags = MDB_TXN_RDONLY;
 		/* Setup db info */
 		txn->mt_numdbs = env->me_numdbs;
 		txn->mt_dbxs = env->me_dbxs;	/* mostly static anyway */

From 54516639acab87aad156230f8a799e9128d266fe Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Wed, 8 Apr 2015 21:51:16 +0200
Subject: [PATCH 060/504] Renumber MDB_TXN_* flags, simplify flag code

---
 libraries/liblmdb/mdb.c | 46 ++++++++++++++++++++---------------------
 1 file changed, 23 insertions(+), 23 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 8c69e9fa43..0e2e227b0a 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1080,12 +1080,16 @@ struct MDB_txn {
  *	@ingroup internal
  *	@{
  */
-#define MDB_TXN_RDONLY		0x01		/**< read-only transaction */
+	/** #mdb_txn_begin() flags */
+#define MDB_TXN_BEGIN_FLAGS	(MDB_NOMETASYNC|MDB_NOSYNC|MDB_RDONLY)
+#define MDB_TXN_NOMETASYNC	MDB_NOMETASYNC	/**< don't sync meta for this txn on commit */
+#define MDB_TXN_NOSYNC		MDB_NOSYNC	/**< don't sync this txn on commit */
+#define MDB_TXN_RDONLY		MDB_RDONLY	/**< read-only transaction */
+	/* internal txn flags */
+#define MDB_TXN_WRITEMAP	MDB_WRITEMAP	/**< copy of #MDB_env flag in writers */
 #define MDB_TXN_ERROR		0x02		/**< txn is unusable after an error */
 #define MDB_TXN_DIRTY		0x04		/**< must write, even if dirty list is empty */
 #define MDB_TXN_SPILLS		0x08		/**< txn or a parent has spilled pages */
-#define MDB_TXN_NOSYNC		0x10		/**< don't sync this txn on commit */
-#define MDB_TXN_NOMETASYNC	0x20		/**< don't sync meta for this txn on commit */
 /** @} */
 	unsigned int	mt_flags;		/**< @ref mdb_txn */
 	/** #dirty_list room: Array size - \#dirty pages visible to this txn.
@@ -1996,7 +2000,7 @@ mdb_page_dirty(MDB_txn *txn, MDB_page *mp)
 	MDB_ID2 mid;
 	int rc, (*insert)(MDB_ID2L, MDB_ID2 *);
 
-	if (txn->mt_env->me_flags & MDB_WRITEMAP) {
+	if (txn->mt_flags & MDB_TXN_WRITEMAP) {
 		insert = mdb_mid2l_append;
 	} else {
 		insert = mdb_mid2l_insert;
@@ -2570,7 +2574,7 @@ mdb_txn_renew0(MDB_txn *txn)
 	int rc, new_notls = 0;
 
 	if (txn->mt_flags & MDB_TXN_RDONLY) {
-		txn->mt_flags = MDB_TXN_RDONLY;
+		txn->mt_flags &= MDB_TXN_BEGIN_FLAGS;
 		/* Setup db info */
 		txn->mt_numdbs = env->me_numdbs;
 		txn->mt_dbxs = env->me_dbxs;	/* mostly static anyway */
@@ -2720,18 +2724,20 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 	MDB_ntxn *ntxn;
 	int rc, size, tsize = sizeof(MDB_txn);
 
+	flags &= MDB_TXN_BEGIN_FLAGS;
+	flags |= env->me_flags & MDB_WRITEMAP;
+
 	if (env->me_flags & MDB_FATAL_ERROR) {
 		DPUTS("environment had fatal error, must shutdown!");
 		return MDB_PANIC;
 	}
-	if ((env->me_flags & MDB_RDONLY) && !(flags & MDB_RDONLY))
+	if (env->me_flags & MDB_RDONLY & ~flags) /* write txn in RDONLY env */
 		return EACCES;
 	if (parent) {
 		/* Nested transactions: Max 1 child, write txns only, no writemap */
 		if (parent->mt_child ||
-			(flags & MDB_RDONLY) ||
-			(parent->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_ERROR)) ||
-			(env->me_flags & MDB_WRITEMAP))
+			((flags | parent->mt_flags) &
+			 (MDB_RDONLY|MDB_WRITEMAP|MDB_TXN_ERROR)))
 		{
 			return (parent->mt_flags & MDB_TXN_RDONLY) ? EINVAL : MDB_BAD_TXN;
 		}
@@ -2754,14 +2760,9 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 	}
 	txn->mt_dbs = (MDB_db *) ((char *)txn + tsize);
 	if (flags & MDB_RDONLY) {
-		txn->mt_flags |= MDB_TXN_RDONLY;
 		txn->mt_dbflags = (unsigned char *)(txn->mt_dbs + env->me_maxdbs);
 		txn->mt_dbiseqs = env->me_dbiseqs;
 	} else {
-		if (flags & MDB_NOSYNC)
-			txn->mt_flags |= MDB_TXN_NOSYNC;
-		if (flags & MDB_NOMETASYNC)
-			txn->mt_flags |= MDB_TXN_NOMETASYNC;
 		txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs);
 		if (parent) {
 			txn->mt_dbiseqs = parent->mt_dbiseqs;
@@ -2771,6 +2772,7 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 			txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs);
 		}
 	}
+	txn->mt_flags = flags;
 	txn->mt_env = env;
 
 ok:
@@ -2822,7 +2824,7 @@ ok:
 	} else {
 		*ret = txn;
 		DPRINTF(("begin txn %"Z"u%c %p on mdbenv %p, root page %"Z"u",
-			txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w',
+			txn->mt_txnid, (flags & MDB_RDONLY) ? 'r' : 'w',
 			(void *) txn, (void *) env, txn->mt_dbs[MAIN_DBI].md_root));
 	}
 
@@ -3660,6 +3662,7 @@ mdb_env_write_meta(MDB_txn *txn)
 {
 	MDB_env *env;
 	MDB_meta	meta, metab, *mp;
+	unsigned flags;
 	size_t mapsize;
 	off_t off;
 	int rc, len, toggle;
@@ -3676,13 +3679,14 @@ mdb_env_write_meta(MDB_txn *txn)
 		toggle, txn->mt_dbs[MAIN_DBI].md_root));
 
 	env = txn->mt_env;
+	flags = txn->mt_flags & env->me_flags;
 	mp = env->me_metas[toggle];
 	mapsize = env->me_metas[toggle ^ 1]->mm_mapsize;
 	/* Persist any increases of mapsize config */
 	if (mapsize < env->me_mapsize)
 		mapsize = env->me_mapsize;
 
-	if (env->me_flags & MDB_WRITEMAP) {
+	if (flags & MDB_WRITEMAP) {
 		mp->mm_mapsize = mapsize;
 		mp->mm_dbs[0] = txn->mt_dbs[0];
 		mp->mm_dbs[1] = txn->mt_dbs[1];
@@ -3692,9 +3696,7 @@ mdb_env_write_meta(MDB_txn *txn)
 		__sync_synchronize();
 #endif
 		mp->mm_txnid = txn->mt_txnid;
-		if (txn->mt_flags & (MDB_TXN_NOSYNC|MDB_TXN_NOMETASYNC))
-			goto done;
-		if (!(env->me_flags & (MDB_NOMETASYNC|MDB_NOSYNC))) {
+		if (!(flags & (MDB_NOMETASYNC|MDB_NOSYNC))) {
 			unsigned meta_size = env->me_psize;
 			rc = (env->me_flags & MDB_MAPASYNC) ? MS_ASYNC : MS_SYNC;
 			ptr = env->me_map;
@@ -3730,9 +3732,7 @@ mdb_env_write_meta(MDB_txn *txn)
 	off += PAGEHDRSZ;
 
 	/* Write to the SYNC fd */
-	mfd = ((env->me_flags & (MDB_NOSYNC|MDB_NOMETASYNC)) ||
-		(txn->mt_flags & (MDB_TXN_NOSYNC|MDB_TXN_NOMETASYNC))) ?
-		env->me_fd : env->me_mfd;
+	mfd = (flags & (MDB_NOSYNC|MDB_NOMETASYNC)) ? env->me_fd : env->me_mfd;
 #ifdef _WIN32
 	{
 		memset(&ov, 0, sizeof(ov));
@@ -5133,7 +5133,7 @@ mdb_page_get(MDB_txn *txn, pgno_t pgno, MDB_page **ret, int *lvl)
 	MDB_page *p = NULL;
 	int level;
 
-	if (!((txn->mt_flags & MDB_TXN_RDONLY) | (env->me_flags & MDB_WRITEMAP))) {
+	if (! (txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_WRITEMAP))) {
 		MDB_txn *tx2 = txn;
 		level = 1;
 		do {

From 8adee9464f65d1702b81a7c604f1a48baa7a0ad5 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Wed, 8 Apr 2015 21:51:50 +0200
Subject: [PATCH 061/504] Fix per-txn MDB_NOMETASYNC, MDB_NOSYNC.

Fallout from 4d02c741b120786df1b87ee9ed49c1d3f9bc7522. The flags
were ignored: mdb_txn_renew0/begin cleared and/or did not set them.
---
 libraries/liblmdb/mdb.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 0e2e227b0a..4a5b7f086a 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2735,9 +2735,9 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 		return EACCES;
 	if (parent) {
 		/* Nested transactions: Max 1 child, write txns only, no writemap */
+		flags |= parent->mt_flags;
 		if (parent->mt_child ||
-			((flags | parent->mt_flags) &
-			 (MDB_RDONLY|MDB_WRITEMAP|MDB_TXN_ERROR)))
+			(flags & (MDB_RDONLY|MDB_WRITEMAP|MDB_TXN_ERROR)))
 		{
 			return (parent->mt_flags & MDB_TXN_RDONLY) ? EINVAL : MDB_BAD_TXN;
 		}
@@ -2794,7 +2794,6 @@ ok:
 		parent->mt_child = txn;
 		txn->mt_parent = parent;
 		txn->mt_numdbs = parent->mt_numdbs;
-		txn->mt_flags = parent->mt_flags;
 		txn->mt_dbxs = parent->mt_dbxs;
 		memcpy(txn->mt_dbs, parent->mt_dbs, txn->mt_numdbs * sizeof(MDB_db));
 		/* Copy parent's mt_dbflags, but clear DB_NEW */
@@ -2822,6 +2821,7 @@ ok:
 		if (txn != env->me_txn0)
 			free(txn);
 	} else {
+		txn->mt_flags |= flags;	/* for txn==me_txn0, no effect otherwise */
 		*ret = txn;
 		DPRINTF(("begin txn %"Z"u%c %p on mdbenv %p, root page %"Z"u",
 			txn->mt_txnid, (flags & MDB_RDONLY) ? 'r' : 'w',

From 6a86f22b2ac4cf982efd63c22fbe4dccf81adc73 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Wed, 8 Apr 2015 21:52:05 +0200
Subject: [PATCH 062/504] Cleanup MDB_env.me_txn0.

More fallout from 4d02c741b120786df1b87ee9ed49c1d3f9bc7522.
---
 libraries/liblmdb/mdb.c | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 4a5b7f086a..cfce8a1e0c 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2640,6 +2640,7 @@ mdb_txn_renew0(MDB_txn *txn)
 			meta = env->me_metas[txn->mt_txnid & 1];
 		}
 	} else {
+		/* Not yet touching txn == env->me_txn0, it may be active */
 		if (ti) {
 			if (LOCK_MUTEX(rc, env, MDB_MUTEX(env, w)))
 				return rc;
@@ -2722,7 +2723,7 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 {
 	MDB_txn *txn;
 	MDB_ntxn *ntxn;
-	int rc, size, tsize = sizeof(MDB_txn);
+	int rc, size, tsize;
 
 	flags &= MDB_TXN_BEGIN_FLAGS;
 	flags |= env->me_flags & MDB_WRITEMAP;
@@ -2733,6 +2734,8 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 	}
 	if (env->me_flags & MDB_RDONLY & ~flags) /* write txn in RDONLY env */
 		return EACCES;
+
+	size = tsize = sizeof(MDB_txn);
 	if (parent) {
 		/* Nested transactions: Max 1 child, write txns only, no writemap */
 		flags |= parent->mt_flags;
@@ -2741,16 +2744,15 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 		{
 			return (parent->mt_flags & MDB_TXN_RDONLY) ? EINVAL : MDB_BAD_TXN;
 		}
-		tsize = sizeof(MDB_ntxn);
-	}
-	size = tsize;
-	if (!(flags & MDB_RDONLY)) {
-		if (!parent) {
-			txn = env->me_txn0;	/* just reuse preallocated write txn */
-			goto ok;
-		}
-		/* child txns use own copy of cursors */
+		/* Child txns save MDB_pgstate and use own copy of cursors */
+		size = tsize = sizeof(MDB_ntxn);
 		size += env->me_maxdbs * sizeof(MDB_cursor *);
+	} else if (!(flags & MDB_RDONLY)) {
+		/* Reuse preallocated write txn. However, do not touch it until
+		 * mdb_txn_renew0() succeeds, since it currently may be active.
+		 */
+		txn = env->me_txn0;
+		goto renew;
 	}
 	size += env->me_maxdbs * (sizeof(MDB_db)+1);
 
@@ -2775,7 +2777,6 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 	txn->mt_flags = flags;
 	txn->mt_env = env;
 
-ok:
 	if (parent) {
 		unsigned int i;
 		txn->mt_u.dirty_list = malloc(sizeof(MDB_ID2)*MDB_IDL_UM_SIZE);
@@ -2815,6 +2816,7 @@ ok:
 		if (rc)
 			mdb_txn_reset0(txn, "beginchild-fail");
 	} else {
+renew:
 		rc = mdb_txn_renew0(txn);
 	}
 	if (rc) {
@@ -4732,15 +4734,13 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 			if (rc)
 				goto leave;
 		}
-		if (!((flags & MDB_RDONLY) ||
-			  (env->me_pbuf = calloc(1, env->me_psize))))
-			rc = ENOMEM;
 		if (!(flags & MDB_RDONLY)) {
 			MDB_txn *txn;
 			int tsize = sizeof(MDB_txn), size = tsize + env->me_maxdbs *
 				(sizeof(MDB_db)+sizeof(MDB_cursor *)+sizeof(unsigned int)+1);
-			txn = calloc(1, size);
-			if (txn) {
+			if ((env->me_pbuf = calloc(1, env->me_psize)) &&
+				(txn = calloc(1, size)))
+			{
 				txn->mt_dbs = (MDB_db *)((char *)txn + tsize);
 				txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs);
 				txn->mt_dbiseqs = (unsigned int *)(txn->mt_cursors + env->me_maxdbs);

From 5224774f3ccf44d7428350bb28a4e9eec7e7c7cf Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 14 Apr 2015 20:24:30 +0200
Subject: [PATCH 063/504] Reformat mdb_dbi_open() doc for clarity

---
 libraries/liblmdb/lmdb.h | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 736d19033b..d67026c79e 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -1043,14 +1043,16 @@ int  mdb_txn_renew(MDB_txn *txn);
 	 * The database handle may be discarded by calling #mdb_dbi_close().
 	 * The old database handle is returned if the database was already open.
 	 * The handle may only be closed once.
+	 *
 	 * The database handle will be private to the current transaction until
 	 * the transaction is successfully committed. If the transaction is
 	 * aborted the handle will be closed automatically.
-	 * After a successful commit the
-	 * handle will reside in the shared environment, and may be used
-	 * by other transactions. This function must not be called from
-	 * multiple concurrent transactions in the same process. A transaction
-	 * that uses this function must finish (either commit or abort) before
+	 * After a successful commit the handle will reside in the shared
+	 * environment, and may be used by other transactions.
+	 *
+	 * This function must not be called from multiple concurrent
+	 * transactions in the same process. A transaction that uses
+	 * this function must finish (either commit or abort) before
 	 * any other transaction in the process may use this function.
 	 *
 	 * To use named databases (with name != NULL), #mdb_env_set_maxdbs()

From 2f6877ff7bccbbd550387cb343687e7dcf72a406 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 14 Apr 2015 20:25:51 +0200
Subject: [PATCH 064/504] Cleanup mtest* and sample-*.

Fix mdb_txn_begin(&read-only txn) calls. Test mdb_env_set_maxreaders().
Rename DBI open/close functions. Move mdb_dbi_close() out of txn.
---
 libraries/liblmdb/mtest.c        | 12 +++++++-----
 libraries/liblmdb/mtest2.c       | 13 +++++++------
 libraries/liblmdb/mtest3.c       | 12 ++++++------
 libraries/liblmdb/mtest4.c       | 12 ++++++------
 libraries/liblmdb/mtest5.c       | 12 ++++++------
 libraries/liblmdb/mtest6.c       |  9 +++++----
 libraries/liblmdb/sample-mdb.txt |  4 ++--
 7 files changed, 39 insertions(+), 35 deletions(-)

diff --git a/libraries/liblmdb/mtest.c b/libraries/liblmdb/mtest.c
index 66dabc752b..7efa8b59e1 100644
--- a/libraries/liblmdb/mtest.c
+++ b/libraries/liblmdb/mtest.c
@@ -45,10 +45,12 @@ int main(int argc,char * argv[])
 	    }
     
 		E(mdb_env_create(&env));
+		E(mdb_env_set_maxreaders(env, 1));
 		E(mdb_env_set_mapsize(env, 10485760));
 		E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP /*|MDB_NOSYNC*/, 0664));
+
 		E(mdb_txn_begin(env, NULL, 0, &txn));
-		E(mdb_open(txn, NULL, 0, &dbi));
+		E(mdb_dbi_open(txn, NULL, 0, &dbi));
    
 		key.mv_size = sizeof(int);
 		key.mv_data = sval;
@@ -68,7 +70,7 @@ int main(int argc,char * argv[])
 		E(mdb_txn_commit(txn));
 		E(mdb_env_stat(env, &mst));
 
-		E(mdb_txn_begin(env, NULL, 1, &txn));
+		E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 		E(mdb_cursor_open(txn, dbi, &cursor));
 		while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
 			printf("key: %p %.*s, data: %p %.*s\n",
@@ -97,7 +99,7 @@ int main(int argc,char * argv[])
 		printf("Deleted %d values\n", j);
 
 		E(mdb_env_stat(env, &mst));
-		E(mdb_txn_begin(env, NULL, 1, &txn));
+		E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 		E(mdb_cursor_open(txn, dbi, &cursor));
 		printf("Cursor next\n");
 		while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
@@ -164,9 +166,9 @@ int main(int argc,char * argv[])
 				data.mv_data, (int) data.mv_size, (char *) data.mv_data);
 		}
 		mdb_cursor_close(cursor);
-		mdb_close(env, dbi);
-
 		mdb_txn_abort(txn);
+
+		mdb_dbi_close(env, dbi);
 		mdb_env_close(env);
 
 	return 0;
diff --git a/libraries/liblmdb/mtest2.c b/libraries/liblmdb/mtest2.c
index 33e0e741c5..cc6ecf6026 100644
--- a/libraries/liblmdb/mtest2.c
+++ b/libraries/liblmdb/mtest2.c
@@ -47,11 +47,13 @@ int main(int argc,char * argv[])
 	}
 
 	E(mdb_env_create(&env));
+	E(mdb_env_set_maxreaders(env, 1));
 	E(mdb_env_set_mapsize(env, 10485760));
 	E(mdb_env_set_maxdbs(env, 4));
 	E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
+
 	E(mdb_txn_begin(env, NULL, 0, &txn));
-	E(mdb_open(txn, "id1", MDB_CREATE, &dbi));
+	E(mdb_dbi_open(txn, "id1", MDB_CREATE, &dbi));
    
 	key.mv_size = sizeof(int);
 	key.mv_data = sval;
@@ -68,7 +70,7 @@ int main(int argc,char * argv[])
 	E(mdb_txn_commit(txn));
 	E(mdb_env_stat(env, &mst));
 
-	E(mdb_txn_begin(env, NULL, 1, &txn));
+	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
 		printf("key: %p %.*s, data: %p %.*s\n",
@@ -97,7 +99,7 @@ int main(int argc,char * argv[])
 	printf("Deleted %d values\n", j);
 
 	E(mdb_env_stat(env, &mst));
-	E(mdb_txn_begin(env, NULL, 1, &txn));
+	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 	printf("Cursor next\n");
 	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
@@ -114,10 +116,9 @@ int main(int argc,char * argv[])
 	}
 	CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
 	mdb_cursor_close(cursor);
-	mdb_close(env, dbi);
-
 	mdb_txn_abort(txn);
-	mdb_env_close(env);
 
+	mdb_dbi_close(env, dbi);
+	mdb_env_close(env);
 	return 0;
 }
diff --git a/libraries/liblmdb/mtest3.c b/libraries/liblmdb/mtest3.c
index 85872c67ed..9db79e625d 100644
--- a/libraries/liblmdb/mtest3.c
+++ b/libraries/liblmdb/mtest3.c
@@ -53,8 +53,9 @@ int main(int argc,char * argv[])
 	E(mdb_env_set_mapsize(env, 10485760));
 	E(mdb_env_set_maxdbs(env, 4));
 	E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
+
 	E(mdb_txn_begin(env, NULL, 0, &txn));
-	E(mdb_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi));
+	E(mdb_dbi_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi));
 
 	key.mv_size = sizeof(int);
 	key.mv_data = kval;
@@ -73,7 +74,7 @@ int main(int argc,char * argv[])
 	E(mdb_txn_commit(txn));
 	E(mdb_env_stat(env, &mst));
 
-	E(mdb_txn_begin(env, NULL, 1, &txn));
+	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
 		printf("key: %p %.*s, data: %p %.*s\n",
@@ -107,7 +108,7 @@ int main(int argc,char * argv[])
 	printf("Deleted %d values\n", j);
 
 	E(mdb_env_stat(env, &mst));
-	E(mdb_txn_begin(env, NULL, 1, &txn));
+	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 	printf("Cursor next\n");
 	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
@@ -124,10 +125,9 @@ int main(int argc,char * argv[])
 	}
 	CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
 	mdb_cursor_close(cursor);
-	mdb_close(env, dbi);
-
 	mdb_txn_abort(txn);
-	mdb_env_close(env);
 
+	mdb_dbi_close(env, dbi);
+	mdb_env_close(env);
 	return 0;
 }
diff --git a/libraries/liblmdb/mtest4.c b/libraries/liblmdb/mtest4.c
index fdcd46a373..6df890e2d7 100644
--- a/libraries/liblmdb/mtest4.c
+++ b/libraries/liblmdb/mtest4.c
@@ -51,8 +51,9 @@ int main(int argc,char * argv[])
 	E(mdb_env_set_mapsize(env, 10485760));
 	E(mdb_env_set_maxdbs(env, 4));
 	E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
+
 	E(mdb_txn_begin(env, NULL, 0, &txn));
-	E(mdb_open(txn, "id4", MDB_CREATE|MDB_DUPSORT|MDB_DUPFIXED, &dbi));
+	E(mdb_dbi_open(txn, "id4", MDB_CREATE|MDB_DUPSORT|MDB_DUPFIXED, &dbi));
 
 	key.mv_size = sizeof(int);
 	key.mv_data = kval;
@@ -72,7 +73,7 @@ int main(int argc,char * argv[])
 
 	/* there should be one full page of dups now.
 	 */
-	E(mdb_txn_begin(env, NULL, 1, &txn));
+	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
 		printf("key: %p %.*s, data: %p %.*s\n",
@@ -142,7 +143,7 @@ int main(int argc,char * argv[])
 	printf("Deleted %d values\n", j);
 
 	E(mdb_env_stat(env, &mst));
-	E(mdb_txn_begin(env, NULL, 1, &txn));
+	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 	printf("Cursor next\n");
 	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
@@ -159,10 +160,9 @@ int main(int argc,char * argv[])
 	}
 	CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
 	mdb_cursor_close(cursor);
-	mdb_close(env, dbi);
-
 	mdb_txn_abort(txn);
-	mdb_env_close(env);
 
+	mdb_dbi_close(env, dbi);
+	mdb_env_close(env);
 	return 0;
 }
diff --git a/libraries/liblmdb/mtest5.c b/libraries/liblmdb/mtest5.c
index 5295bce41f..14e3c0da4a 100644
--- a/libraries/liblmdb/mtest5.c
+++ b/libraries/liblmdb/mtest5.c
@@ -53,8 +53,9 @@ int main(int argc,char * argv[])
 	E(mdb_env_set_mapsize(env, 10485760));
 	E(mdb_env_set_maxdbs(env, 4));
 	E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
+
 	E(mdb_txn_begin(env, NULL, 0, &txn));
-	E(mdb_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi));
+	E(mdb_dbi_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 
 	key.mv_size = sizeof(int);
@@ -75,7 +76,7 @@ int main(int argc,char * argv[])
 	E(mdb_txn_commit(txn));
 	E(mdb_env_stat(env, &mst));
 
-	E(mdb_txn_begin(env, NULL, 1, &txn));
+	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
 		printf("key: %p %.*s, data: %p %.*s\n",
@@ -109,7 +110,7 @@ int main(int argc,char * argv[])
 	printf("Deleted %d values\n", j);
 
 	E(mdb_env_stat(env, &mst));
-	E(mdb_txn_begin(env, NULL, 1, &txn));
+	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 	printf("Cursor next\n");
 	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
@@ -126,10 +127,9 @@ int main(int argc,char * argv[])
 	}
 	CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
 	mdb_cursor_close(cursor);
-	mdb_close(env, dbi);
-
 	mdb_txn_abort(txn);
-	mdb_env_close(env);
 
+	mdb_dbi_close(env, dbi);
+	mdb_env_close(env);
 	return 0;
 }
diff --git a/libraries/liblmdb/mtest6.c b/libraries/liblmdb/mtest6.c
index b351c1a06a..174f4f67f6 100644
--- a/libraries/liblmdb/mtest6.c
+++ b/libraries/liblmdb/mtest6.c
@@ -46,8 +46,9 @@ int main(int argc,char * argv[])
 	E(mdb_env_set_mapsize(env, 10485760));
 	E(mdb_env_set_maxdbs(env, 4));
 	E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
+
 	E(mdb_txn_begin(env, NULL, 0, &txn));
-	E(mdb_open(txn, "id6", MDB_CREATE|MDB_INTEGERKEY, &dbi));
+	E(mdb_dbi_open(txn, "id6", MDB_CREATE|MDB_INTEGERKEY, &dbi));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 	E(mdb_stat(txn, dbi, &mst));
 
@@ -110,7 +111,7 @@ int main(int argc,char * argv[])
 	printf("Deleted %d values\n", j);
 
 	E(mdb_env_stat(env, &mst));
-	E(mdb_txn_begin(env, NULL, 1, &txn));
+	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 	printf("Cursor next\n");
 	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
@@ -127,9 +128,9 @@ int main(int argc,char * argv[])
 	}
 	CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
 	mdb_cursor_close(cursor);
-	mdb_close(env, dbi);
-
 	mdb_txn_abort(txn);
+
+	mdb_dbi_close(env, dbi);
 #endif
 	mdb_env_close(env);
 
diff --git a/libraries/liblmdb/sample-mdb.txt b/libraries/liblmdb/sample-mdb.txt
index d311f8e954..10a256870b 100644
--- a/libraries/liblmdb/sample-mdb.txt
+++ b/libraries/liblmdb/sample-mdb.txt
@@ -32,7 +32,7 @@ int main(int argc,char * argv[])
 	rc = mdb_env_create(&env);
 	rc = mdb_env_open(env, "./testdb", 0, 0664);
 	rc = mdb_txn_begin(env, NULL, 0, &txn);
-	rc = mdb_open(txn, NULL, 0, &dbi);
+	rc = mdb_dbi_open(txn, NULL, 0, &dbi);
 
 	key.mv_size = sizeof(int);
 	key.mv_data = sval;
@@ -56,7 +56,7 @@ int main(int argc,char * argv[])
 	mdb_cursor_close(cursor);
 	mdb_txn_abort(txn);
 leave:
-	mdb_close(env, dbi);
+	mdb_dbi_close(env, dbi);
 	mdb_env_close(env);
 	return 0;
 }

From b0fca5b956daf89ca5c7057ae179f626c1c5e0a0 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 15 Apr 2015 23:20:55 +0100
Subject: [PATCH 065/504] ITS#8062 fix rebalance

(Probably fixes the ITS, definitely fixes a bug) when collapsing
the root page, fixups of other cursors was incomplete.
---
 libraries/liblmdb/mdb.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index cfce8a1e0c..36898d639f 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7837,12 +7837,12 @@ mdb_rebalance(MDB_cursor *mc)
 						m3 = m2;
 					if (m3 == mc || m3->mc_snum < mc->mc_snum) continue;
 					if (m3->mc_pg[0] == mp) {
-						m3->mc_snum--;
-						m3->mc_top--;
 						for (i=0; i<m3->mc_snum; i++) {
 							m3->mc_pg[i] = m3->mc_pg[i+1];
 							m3->mc_ki[i] = m3->mc_ki[i+1];
 						}
+						m3->mc_snum--;
+						m3->mc_top--;
 					}
 				}
 			}
@@ -7912,7 +7912,11 @@ mdb_rebalance(MDB_cursor *mc)
 		} else {
 			oldki += NUMKEYS(mn.mc_pg[mn.mc_top]);
 			mn.mc_ki[mn.mc_top] += mc->mc_ki[mn.mc_top] + 1;
+			/* We want mdb_rebalance to find mn when doing fixups */
+			mn.mc_next = mc->mc_txn->mt_cursors[mc->mc_dbi];
+			mc->mc_txn->mt_cursors[mc->mc_dbi] = &mn;
 			rc = mdb_page_merge(mc, &mn);
+			mc->mc_txn->mt_cursors[mc->mc_dbi] = mn.mc_next;
 			mdb_cursor_copy(&mn, mc);
 		}
 		mc->mc_flags &= ~C_EOF;

From ccf4d8b385117aab1e30b46bfa0d887f621b9423 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 16 Apr 2015 00:19:40 +0100
Subject: [PATCH 066/504] ITS#8062 also handle subcursors

---
 libraries/liblmdb/mdb.c | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 36898d639f..a95c7cdb5c 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7910,13 +7910,23 @@ mdb_rebalance(MDB_cursor *mc)
 		if (mc->mc_ki[ptop] == 0) {
 			rc = mdb_page_merge(&mn, mc);
 		} else {
+			MDB_cursor dummy;
 			oldki += NUMKEYS(mn.mc_pg[mn.mc_top]);
 			mn.mc_ki[mn.mc_top] += mc->mc_ki[mn.mc_top] + 1;
 			/* We want mdb_rebalance to find mn when doing fixups */
-			mn.mc_next = mc->mc_txn->mt_cursors[mc->mc_dbi];
-			mc->mc_txn->mt_cursors[mc->mc_dbi] = &mn;
+			if (mc->mc_flags & C_SUB) {
+				dummy.mc_next = mc->mc_txn->mt_cursors[mc->mc_dbi];
+				mc->mc_txn->mt_cursors[mc->mc_dbi] = &dummy;
+				dummy.mc_xcursor = (MDB_xcursor *)&mn;
+			} else {
+				mn.mc_next = mc->mc_txn->mt_cursors[mc->mc_dbi];
+				mc->mc_txn->mt_cursors[mc->mc_dbi] = &mn;
+			}
 			rc = mdb_page_merge(mc, &mn);
-			mc->mc_txn->mt_cursors[mc->mc_dbi] = mn.mc_next;
+			if (mc->mc_flags & C_SUB)
+				mc->mc_txn->mt_cursors[mc->mc_dbi] = dummy.mc_next;
+			else
+				mc->mc_txn->mt_cursors[mc->mc_dbi] = mn.mc_next;
 			mdb_cursor_copy(&mn, mc);
 		}
 		mc->mc_flags &= ~C_EOF;

From 4c62fde880314f23871a68c8eabe8551a2aaf54b Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Fri, 17 Apr 2015 18:32:54 +0100
Subject: [PATCH 067/504] ITS#8106 retry writes on EINTR

---
 libraries/liblmdb/mdb.c | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index a95c7cdb5c..4e0607f9da 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -3236,6 +3236,7 @@ mdb_page_flush(MDB_txn *txn, int keep)
 		/* Write up to MDB_COMMIT_PAGES dirty pages at a time. */
 		if (pos!=next_pos || n==MDB_COMMIT_PAGES || wsize+size>MAX_WRITE) {
 			if (n) {
+retry_write:
 				/* Write previous page(s) */
 #ifdef MDB_USE_PWRITEV
 				wres = pwritev(env->me_fd, iov, n, wpos);
@@ -3243,8 +3244,11 @@ mdb_page_flush(MDB_txn *txn, int keep)
 				if (n == 1) {
 					wres = pwrite(env->me_fd, iov[0].iov_base, wsize, wpos);
 				} else {
+retry_seek:
 					if (lseek(env->me_fd, wpos, SEEK_SET) == -1) {
 						rc = ErrCode();
+						if (rc == EINTR)
+							goto retry_seek;
 						DPRINTF(("lseek: %s", strerror(rc)));
 						return rc;
 					}
@@ -3254,6 +3258,8 @@ mdb_page_flush(MDB_txn *txn, int keep)
 				if (wres != wsize) {
 					if (wres < 0) {
 						rc = ErrCode();
+						if (rc == EINTR)
+							goto retry_write;
 						DPRINTF(("Write error: %s", strerror(rc)));
 					} else {
 						rc = EIO; /* TODO: Use which error code? */
@@ -3627,7 +3633,8 @@ mdb_env_init_meta(MDB_env *env, MDB_meta *meta)
 	int len;
 #define DO_PWRITE(rc, fd, ptr, size, len, pos)	do { \
 	len = pwrite(fd, ptr, size, pos);	\
-	rc = (len >= 0); } while(0)
+	if (len == -1 && ErrCode() == EINTR) continue; \
+	rc = (len >= 0); break; } while(1)
 #endif
 
 	DPUTS("writing new meta page");
@@ -3735,6 +3742,7 @@ mdb_env_write_meta(MDB_txn *txn)
 
 	/* Write to the SYNC fd */
 	mfd = (flags & (MDB_NOSYNC|MDB_NOMETASYNC)) ? env->me_fd : env->me_mfd;
+retry_write:
 #ifdef _WIN32
 	{
 		memset(&ov, 0, sizeof(ov));
@@ -3747,6 +3755,8 @@ mdb_env_write_meta(MDB_txn *txn)
 #endif
 	if (rc != len) {
 		rc = rc < 0 ? ErrCode() : EIO;
+		if (rc == EINTR)
+			goto retry_write;
 		DPUTS("write failed, disk error?");
 		/* On a failure, the pagecache still contains the new data.
 		 * Write some old data back, to prevent it from being used.

From f4cab2994f45b56fd5ffa1f332850b0e3285b665 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 23 Apr 2015 05:49:56 +0100
Subject: [PATCH 068/504] ITS#8109 fix mdb_cursor_del0 on empty DB

---
 libraries/liblmdb/mdb.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 4e0607f9da..f1c7508ccf 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7963,6 +7963,13 @@ mdb_cursor_del0(MDB_cursor *mc)
 		MDB_cursor *m2, *m3;
 		MDB_dbi dbi = mc->mc_dbi;
 
+		/* DB is totally empty now, just bail out.
+		 * Other cursors adjustments were already done
+		 * by mdb_rebalance and aren't needed here.
+		 */
+		if (!mc->mc_snum)
+			return rc;
+
 		mp = mc->mc_pg[mc->mc_top];
 		nkeys = NUMKEYS(mp);
 

From ed13d4aa59cf06a1bb6b096fe19af63fc6f8737a Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 29 Apr 2015 17:12:40 +0100
Subject: [PATCH 069/504] ITS#8117 fix INTEGERDUP compare

---
 libraries/liblmdb/mdb.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index f1c7508ccf..892dd044b1 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7194,6 +7194,12 @@ mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node)
 				mx->mx_db.md_flags |= MDB_INTEGERKEY;
 		}
 	}
+#if UINT_MAX < SIZE_MAX
+	if (mc->mc_dbx->md_dcmp == mdb_cmp_int && mx->mx_db.md_pad == sizeof(size_t)) {
+		mc->mc_dbx->md_dcmp = mdb_cmp_clong;
+		mx->mx_dbx.md_cmp = mdb_cmp_clong;
+	}
+#endif
 	DPRINTF(("Sub-db -%u root page %"Z"u", mx->mx_cursor.mc_dbi,
 		mx->mx_db.md_root));
 	mx->mx_dbflag = DB_VALID|DB_DIRTY; /* DB_DIRTY guides mdb_cursor_touch */

From ce7ba0ceb69fd2cd56d557c1a877d046ec9f246c Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 29 Apr 2015 17:28:21 +0100
Subject: [PATCH 070/504] ITS#8117 cleanup prev commit

---
 libraries/liblmdb/mdb.c | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 892dd044b1..f6c293c6bb 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7203,10 +7203,6 @@ mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node)
 	DPRINTF(("Sub-db -%u root page %"Z"u", mx->mx_cursor.mc_dbi,
 		mx->mx_db.md_root));
 	mx->mx_dbflag = DB_VALID|DB_DIRTY; /* DB_DIRTY guides mdb_cursor_touch */
-#if UINT_MAX < SIZE_MAX
-	if (mx->mx_dbx.md_cmp == mdb_cmp_int && mx->mx_db.md_pad == sizeof(size_t))
-		mx->mx_dbx.md_cmp = mdb_cmp_clong;
-#endif
 }
 
 /** Initialize a cursor for a given transaction and database. */

From 3d2202f31c14c6188093a492a97371069a732557 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sun, 3 May 2015 07:43:11 +0100
Subject: [PATCH 071/504] Revert "ITS#8117 cleanup prev commit"

This reverts commit ce7ba0ceb69fd2cd56d557c1a877d046ec9f246c.

Revert "ITS#8117 fix INTEGERDUP compare"

This reverts commit ed13d4aa59cf06a1bb6b096fe19af63fc6f8737a.
---
 libraries/liblmdb/mdb.c | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index f6c293c6bb..f1c7508ccf 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7194,15 +7194,13 @@ mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node)
 				mx->mx_db.md_flags |= MDB_INTEGERKEY;
 		}
 	}
-#if UINT_MAX < SIZE_MAX
-	if (mc->mc_dbx->md_dcmp == mdb_cmp_int && mx->mx_db.md_pad == sizeof(size_t)) {
-		mc->mc_dbx->md_dcmp = mdb_cmp_clong;
-		mx->mx_dbx.md_cmp = mdb_cmp_clong;
-	}
-#endif
 	DPRINTF(("Sub-db -%u root page %"Z"u", mx->mx_cursor.mc_dbi,
 		mx->mx_db.md_root));
 	mx->mx_dbflag = DB_VALID|DB_DIRTY; /* DB_DIRTY guides mdb_cursor_touch */
+#if UINT_MAX < SIZE_MAX
+	if (mx->mx_dbx.md_cmp == mdb_cmp_int && mx->mx_db.md_pad == sizeof(size_t))
+		mx->mx_dbx.md_cmp = mdb_cmp_clong;
+#endif
 }
 
 /** Initialize a cursor for a given transaction and database. */

From 329e12e4c5723276d1dd09a283a8149744268baf Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sun, 3 May 2015 07:52:47 +0100
Subject: [PATCH 072/504] ITS#8117 better fix

Don't change mc_dbx because we would need to undo the change if
the txn aborts. Make the fix (for get) match existing code for put.
---
 libraries/liblmdb/mdb.c | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index f1c7508ccf..ec8f8f38d3 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5871,15 +5871,21 @@ set1:
 					return rc;
 			}
 		} else if (op == MDB_GET_BOTH || op == MDB_GET_BOTH_RANGE) {
-			MDB_val d2;
-			if ((rc = mdb_node_read(mc->mc_txn, leaf, &d2)) != MDB_SUCCESS)
+			MDB_val olddata;
+			MDB_cmp_func *dcmp;
+			if ((rc = mdb_node_read(mc->mc_txn, leaf, &olddata)) != MDB_SUCCESS)
 				return rc;
-			rc = mc->mc_dbx->md_dcmp(data, &d2);
+			dcmp = mc->mc_dbx->md_dcmp;
+#if UINT_MAX < SIZE_MAX
+			if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t))
+				dcmp = mdb_cmp_clong;
+#endif
+			rc = dcmp(data, &olddata);
 			if (rc) {
 				if (op == MDB_GET_BOTH || rc > 0)
 					return MDB_NOTFOUND;
 				rc = 0;
-				*data = d2;
+				*data = olddata;
 			}
 
 		} else {
@@ -6388,16 +6394,17 @@ more:
 
 			/* Was a single item before, must convert now */
 			if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) {
+				MDB_cmp_func *dcmp;
 				/* Just overwrite the current item */
 				if (flags == MDB_CURRENT)
 					goto current;
-
+				dcmp = mc->mc_dbx->md_dcmp;
 #if UINT_MAX < SIZE_MAX
-				if (mc->mc_dbx->md_dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t))
-					mc->mc_dbx->md_dcmp = mdb_cmp_clong;
+				if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t))
+					dcmp = mdb_cmp_clong;
 #endif
 				/* does data match? */
-				if (!mc->mc_dbx->md_dcmp(data, &olddata)) {
+				if (!dcmp(data, &olddata)) {
 					if (flags & MDB_NODUPDATA)
 						return MDB_KEYEXIST;
 					/* overwrite it */

From 9bb915675afe3acb616b2df2e241adddd6ee432f Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 19 May 2015 20:33:35 +0200
Subject: [PATCH 073/504] ITS#8117 Fix MDB_INTEGERDUP keysize doc + md_dcmp

---
 libraries/liblmdb/lmdb.h | 14 +++++++-------
 libraries/liblmdb/mdb.c  | 27 ++++++++++++++++++---------
 2 files changed, 25 insertions(+), 16 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index d67026c79e..625d02166f 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -303,12 +303,12 @@ typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *rel
 #define MDB_REVERSEKEY	0x02
 	/** use sorted duplicates */
 #define MDB_DUPSORT		0x04
-	/** numeric keys in native byte order.
+	/** numeric keys in native byte order: either unsigned int or size_t.
 	 *  The keys must all be of the same size. */
 #define MDB_INTEGERKEY	0x08
 	/** with #MDB_DUPSORT, sorted dup items have fixed size */
 #define MDB_DUPFIXED	0x10
-	/** with #MDB_DUPSORT, dups are numeric in native byte order */
+	/** with #MDB_DUPSORT, dups are #MDB_INTEGERKEY-style integers */
 #define MDB_INTEGERDUP	0x20
 	/** with #MDB_DUPSORT, use reverse string dups */
 #define MDB_REVERSEDUP	0x40
@@ -1074,9 +1074,9 @@ int  mdb_txn_renew(MDB_txn *txn);
 	 *		keys may have multiple data items, stored in sorted order.) By default
 	 *		keys must be unique and may have only a single data item.
 	 *	<li>#MDB_INTEGERKEY
-	 *		Keys are binary integers in native byte order. Setting this option
-	 *		requires all keys to be the same size, typically sizeof(int)
-	 *		or sizeof(size_t).
+	 *		Keys are binary integers in native byte order, either unsigned int
+	 *		or size_t, and will be sorted as such.
+	 *		The keys must all be of the same size.
 	 *	<li>#MDB_DUPFIXED
 	 *		This flag may only be used in combination with #MDB_DUPSORT. This option
 	 *		tells the library that the data items for this database are all the same
@@ -1084,8 +1084,8 @@ int  mdb_txn_renew(MDB_txn *txn);
 	 *		all data items are the same size, the #MDB_GET_MULTIPLE and #MDB_NEXT_MULTIPLE
 	 *		cursor operations may be used to retrieve multiple items at once.
 	 *	<li>#MDB_INTEGERDUP
-	 *		This option specifies that duplicate data items are also integers, and
-	 *		should be sorted as such.
+	 *		This option specifies that duplicate data items are binary integers,
+	 *		similar to #MDB_INTEGERKEY keys.
 	 *	<li>#MDB_REVERSEDUP
 	 *		This option specifies that duplicate data items should be compared as
 	 *		strings in reverse order.
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index ec8f8f38d3..828c2644fa 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1328,6 +1328,13 @@ static int mdb_reader_check0(MDB_env *env, int rlocked, int *dead);
 static MDB_cmp_func	mdb_cmp_memn, mdb_cmp_memnr, mdb_cmp_int, mdb_cmp_cint, mdb_cmp_long;
 /** @endcond */
 
+/** Compare two items pointing at size_t's of unknown alignment. */
+#ifdef MISALIGNED_OK
+# define mdb_cmp_clong mdb_cmp_long
+#else
+# define mdb_cmp_clong mdb_cmp_cint
+#endif
+
 #ifdef _WIN32
 static SECURITY_DESCRIPTOR mdb_null_sd;
 static SECURITY_ATTRIBUTES mdb_all_sa;
@@ -1641,7 +1648,12 @@ mdb_cmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b)
 int
 mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b)
 {
-	return txn->mt_dbxs[dbi].md_dcmp(a, b);
+	MDB_cmp_func *dcmp = txn->mt_dbxs[dbi].md_dcmp;
+#if UINT_MAX < SIZE_MAX
+	if (dcmp == mdb_cmp_int && a->mv_size == sizeof(size_t))
+		dcmp = mdb_cmp_clong;
+#endif
+	return dcmp(a, b);
 }
 
 /** Allocate memory for a page.
@@ -4891,7 +4903,11 @@ mdb_cmp_long(const MDB_val *a, const MDB_val *b)
 		*(size_t *)a->mv_data > *(size_t *)b->mv_data;
 }
 
-/** Compare two items pointing at aligned unsigned int's */
+/** Compare two items pointing at aligned unsigned int's.
+ *
+ *	This is also set as #MDB_INTEGERDUP|#MDB_DUPFIXED's #MDB_dbx.%md_dcmp,
+ *	but #mdb_cmp_clong() is called instead if the data type is size_t.
+ */
 static int
 mdb_cmp_int(const MDB_val *a, const MDB_val *b)
 {
@@ -4929,13 +4945,6 @@ mdb_cmp_cint(const MDB_val *a, const MDB_val *b)
 #endif
 }
 
-/** Compare two items pointing at size_t's of unknown alignment. */
-#ifdef MISALIGNED_OK
-# define mdb_cmp_clong mdb_cmp_long
-#else
-# define mdb_cmp_clong mdb_cmp_cint
-#endif
-
 /** Compare two items lexically */
 static int
 mdb_cmp_memn(const MDB_val *a, const MDB_val *b)

From 77bdbdd9de439224261c57bdcaf4d3281885f928 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Wed, 20 May 2015 03:17:47 +0200
Subject: [PATCH 074/504] mtest*.c: Fix MDB_NOOVERWRITE, plug cursor leak

---
 libraries/liblmdb/mtest.c  | 6 ++++--
 libraries/liblmdb/mtest2.c | 4 ++--
 libraries/liblmdb/mtest6.c | 9 ++++++---
 3 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/libraries/liblmdb/mtest.c b/libraries/liblmdb/mtest.c
index 7efa8b59e1..9d15088b0c 100644
--- a/libraries/liblmdb/mtest.c
+++ b/libraries/liblmdb/mtest.c
@@ -54,12 +54,13 @@ int main(int argc,char * argv[])
    
 		key.mv_size = sizeof(int);
 		key.mv_data = sval;
-		data.mv_size = sizeof(sval);
-		data.mv_data = sval;
 
 		printf("Adding %d values\n", count);
 	    for (i=0;i<count;i++) {	
 			sprintf(sval, "%03x %d foo bar", values[i], values[i]);
+			/* Set <data> in each iteration, since MDB_NOOVERWRITE may modify it */
+			data.mv_size = sizeof(sval);
+			data.mv_data = sval;
 			if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) {
 				j++;
 				data.mv_size = sizeof(sval);
@@ -130,6 +131,7 @@ int main(int argc,char * argv[])
 				(int) key.mv_size,  (char *) key.mv_data,
 				(int) data.mv_size, (char *) data.mv_data);
 
+		mdb_cursor_close(cursor);
 		mdb_txn_abort(txn);
 
 		printf("Deleting with cursor\n");
diff --git a/libraries/liblmdb/mtest2.c b/libraries/liblmdb/mtest2.c
index cc6ecf6026..eacbe59d53 100644
--- a/libraries/liblmdb/mtest2.c
+++ b/libraries/liblmdb/mtest2.c
@@ -57,12 +57,12 @@ int main(int argc,char * argv[])
    
 	key.mv_size = sizeof(int);
 	key.mv_data = sval;
-	data.mv_size = sizeof(sval);
-	data.mv_data = sval;
 
 	printf("Adding %d values\n", count);
 	for (i=0;i<count;i++) {	
 		sprintf(sval, "%03x %d foo bar", values[i], values[i]);
+		data.mv_size = sizeof(sval);
+		data.mv_data = sval;
 		if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE)))
 			j++;
 	}
diff --git a/libraries/liblmdb/mtest6.c b/libraries/liblmdb/mtest6.c
index 174f4f67f6..ae3c7f264c 100644
--- a/libraries/liblmdb/mtest6.c
+++ b/libraries/liblmdb/mtest6.c
@@ -31,7 +31,7 @@ int main(int argc,char * argv[])
 	int i = 0, j = 0, rc;
 	MDB_env *env;
 	MDB_dbi dbi;
-	MDB_val key, data;
+	MDB_val key, data, sdata;
 	MDB_txn *txn;
 	MDB_stat mst;
 	MDB_cursor *cursor;
@@ -55,25 +55,28 @@ int main(int argc,char * argv[])
 	sval = calloc(1, mst.ms_psize / 4);
 	key.mv_size = sizeof(long);
 	key.mv_data = &kval;
-	data.mv_size = mst.ms_psize / 4 - 30;
-	data.mv_data = sval;
+	sdata.mv_size = mst.ms_psize / 4 - 30;
+	sdata.mv_data = sval;
 
 	printf("Adding 12 values, should yield 3 splits\n");
 	for (i=0;i<12;i++) {
 		kval = i*5;
 		sprintf(sval, "%08x", kval);
+		data = sdata;
 		(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
 	}
 	printf("Adding 12 more values, should yield 3 splits\n");
 	for (i=0;i<12;i++) {
 		kval = i*5+4;
 		sprintf(sval, "%08x", kval);
+		data = sdata;
 		(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
 	}
 	printf("Adding 12 more values, should yield 3 splits\n");
 	for (i=0;i<12;i++) {
 		kval = i*5+1;
 		sprintf(sval, "%08x", kval);
+		data = sdata;
 		(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
 	}
 	E(mdb_cursor_get(cursor, &key, &data, MDB_FIRST));

From 02e8de5edbd13a5512352bbc308359d29870607d Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Wed, 20 May 2015 04:04:38 +0200
Subject: [PATCH 075/504] Set/clear mp_pad, md_pad (MDB_DUPFIXED data size).

mdb_xcursor_init1(): md_pad is only used when MDB_DUPFIXED.
mdb_page_split():    Copy mp_pad too. Used by mdb_page_list().
---
 libraries/liblmdb/mdb.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 828c2644fa..5423575598 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7190,7 +7190,7 @@ mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node)
 		mx->mx_cursor.mc_flags = C_SUB;
 	} else {
 		MDB_page *fp = NODEDATA(node);
-		mx->mx_db.md_pad = mc->mc_pg[mc->mc_top]->mp_pad;
+		mx->mx_db.md_pad = 0;
 		mx->mx_db.md_flags = 0;
 		mx->mx_db.md_depth = 1;
 		mx->mx_db.md_branch_pages = 0;
@@ -8130,6 +8130,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 	/* Create a right sibling. */
 	if ((rc = mdb_page_new(mc, mp->mp_flags, 1, &rp)))
 		return rc;
+	rp->mp_pad = mp->mp_pad;
 	DPRINTF(("new right sibling: page %"Z"u", rp->mp_pgno));
 
 	if (mc->mc_snum < 2) {

From 8d1f6ca7822930d7e1e94739886b6cd50340b7b4 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Wed, 27 May 2015 22:23:50 +0200
Subject: [PATCH 076/504] mdb_txn_renew(): Drop pointless diff from mdb.RE.

Makes the code friendlier to "git cherry-pick".
---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 5423575598..78c2c35150 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2586,7 +2586,7 @@ mdb_txn_renew0(MDB_txn *txn)
 	int rc, new_notls = 0;
 
 	if (txn->mt_flags & MDB_TXN_RDONLY) {
-		txn->mt_flags &= MDB_TXN_BEGIN_FLAGS;
+		txn->mt_flags = MDB_TXN_RDONLY;
 		/* Setup db info */
 		txn->mt_numdbs = env->me_numdbs;
 		txn->mt_dbxs = env->me_dbxs;	/* mostly static anyway */

From aa16d7bc0a0c65b28726aab562bf3ca3c0ef1e5d Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Wed, 27 May 2015 22:26:54 +0200
Subject: [PATCH 077/504] ITS#8157 mdb_txn_renew0(): init after error checks

---
 libraries/liblmdb/mdb.c | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 78c2c35150..612cd0c034 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2581,15 +2581,11 @@ mdb_txn_renew0(MDB_txn *txn)
 	MDB_env *env = txn->mt_env;
 	MDB_txninfo *ti = env->me_txns;
 	MDB_meta *meta;
-	unsigned int i, nr;
+	unsigned int i, nr, flags = txn->mt_flags;
 	uint16_t x;
 	int rc, new_notls = 0;
 
-	if (txn->mt_flags & MDB_TXN_RDONLY) {
-		txn->mt_flags = MDB_TXN_RDONLY;
-		/* Setup db info */
-		txn->mt_numdbs = env->me_numdbs;
-		txn->mt_dbxs = env->me_dbxs;	/* mostly static anyway */
+	if ((flags &= MDB_TXN_RDONLY) != 0) {
 		if (!ti) {
 			meta = env->me_metas[ mdb_env_pick_meta(env) ];
 			txn->mt_txnid = meta->mm_txnid;
@@ -2651,6 +2647,7 @@ mdb_txn_renew0(MDB_txn *txn)
 			txn->mt_u.reader = r;
 			meta = env->me_metas[txn->mt_txnid & 1];
 		}
+		txn->mt_dbxs = env->me_dbxs;	/* mostly static anyway */
 	} else {
 		/* Not yet touching txn == env->me_txn0, it may be active */
 		if (ti) {
@@ -2662,14 +2659,11 @@ mdb_txn_renew0(MDB_txn *txn)
 			meta = env->me_metas[ mdb_env_pick_meta(env) ];
 			txn->mt_txnid = meta->mm_txnid;
 		}
-		/* Setup db info */
-		txn->mt_numdbs = env->me_numdbs;
 		txn->mt_txnid++;
 #if MDB_DEBUG
 		if (txn->mt_txnid == mdb_debug_start)
 			mdb_debug = 1;
 #endif
-		txn->mt_flags = 0;
 		txn->mt_child = NULL;
 		txn->mt_loose_pgs = NULL;
 		txn->mt_loose_count = 0;
@@ -2689,6 +2683,10 @@ mdb_txn_renew0(MDB_txn *txn)
 	/* Moved to here to avoid a data race in read TXNs */
 	txn->mt_next_pgno = meta->mm_last_pg+1;
 
+	txn->mt_flags = flags;
+
+	/* Setup db info */
+	txn->mt_numdbs = env->me_numdbs;
 	for (i=2; i<txn->mt_numdbs; i++) {
 		x = env->me_dbflags[i];
 		txn->mt_dbs[i].md_flags = x & PERSISTENT_FLAGS;

From 579ad14d7c4521754c9942ac39b39b6110338923 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Thu, 28 May 2015 20:51:24 +0200
Subject: [PATCH 078/504] ITS#8156 Fix MDB_MAXKEYSIZE doc

---
 libraries/liblmdb/mdb.c | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 612cd0c034..482b98e6b3 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -510,12 +510,17 @@ static txnid_t mdb_debug_start;
 	/**	The version number for a database's lockfile format. */
 #define MDB_LOCK_VERSION	 ((MDB_DEVEL) ? 999 : 1)
 
-	/**	@brief The max size of a key we can write, or 0 for dynamic max.
+	/**	@brief The max size of a key we can write, or 0 for computed max.
 	 *
-	 *	Define this as 0 to compute the max from the page size.  511
-	 *	is default for backwards compat: liblmdb <= 0.9.10 can break
-	 *	when modifying a DB with keys/dupsort data bigger than its max.
-	 *	#MDB_DEVEL sets the default to 0.
+	 *	This macro should normally be left alone or set to 0.
+	 *	Note that a database with big keys or dupsort data cannot be
+	 *	reliably modified by a liblmdb which uses a smaller max.
+	 *	The default is 511 for backwards compat, or 0 when #MDB_DEVEL.
+	 *
+	 *	Other values are allowed, for backwards compat.  However:
+	 *	A value bigger than the computed max can break if you do not
+	 *	know what you are doing, and liblmdb <= 0.9.10 can break when
+	 *	modifying a DB with keys/dupsort data bigger than its max.
 	 *
 	 *	Data items in an #MDB_DUPSORT database are also limited to
 	 *	this size, since they're actually keys of a sub-DB.  Keys and

From 68f64aa10449dd8b981c0348caec1fe19bcc11ef Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 2 Jun 2015 20:49:33 +0200
Subject: [PATCH 079/504] Clarify mdb_env_open() doc of 'mode' param

---
 libraries/liblmdb/lmdb.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 625d02166f..a2080d6b03 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -595,8 +595,8 @@ int  mdb_env_create(MDB_env **env);
 	 *		reserved in that case.
 	 *		This flag may be changed at any time using #mdb_env_set_flags().
 	 * </ul>
-	 * @param[in] mode The UNIX permissions to set on created files. This parameter
-	 * is ignored on Windows.
+	 * @param[in] mode The UNIX permissions to set on created files and semaphores.
+	 * This parameter is ignored on Windows.
 	 * @return A non-zero error value on failure and 0 on success. Some possible
 	 * errors are:
 	 * <ul>

From e95d57f39c731e2cd27aa8f604db4ad96e1c3049 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 2 Jun 2015 20:51:03 +0200
Subject: [PATCH 080/504] Use ftok() for semget(), and only lower mode bits

---
 libraries/liblmdb/mdb.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 482b98e6b3..cc0944d59b 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -4548,7 +4548,10 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 		if (!env->me_wmutex) goto fail_errno;
 #elif defined(MDB_USE_SYSV_SEM)
 		unsigned short vals[2] = {1, 1};
-		semid = semget(IPC_PRIVATE, 2, mode);
+		key_t key = ftok(lpath, 'M');
+		if (key == -1)
+			goto fail_errno;
+		semid = semget(key, 2, (mode & 0777) | IPC_CREAT);
 		if (semid < 0)
 			goto fail_errno;
 		semu.array = vals;

From bf55ce55451213be69caea225bd3d713ed3aba7e Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Fri, 19 Jun 2015 14:13:43 +0100
Subject: [PATCH 081/504] Bump version to 0.9.15

(not really, since this is the dev/non-release branch)
---
 libraries/liblmdb/lmdb.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index a2080d6b03..e9a01a5cb9 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -191,7 +191,7 @@ typedef int mdb_filehandle_t;
 /** Library minor version */
 #define MDB_VERSION_MINOR	9
 /** Library patch version */
-#define MDB_VERSION_PATCH	14
+#define MDB_VERSION_PATCH	15
 
 /** Combine args a,b,c into a single integer for easy version comparisons */
 #define MDB_VERINT(a,b,c)	(((a) << 24) | ((b) << 16) | (c))
@@ -201,7 +201,7 @@ typedef int mdb_filehandle_t;
 	MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH)
 
 /** The release date of this library version */
-#define MDB_VERSION_DATE	"September 20, 2014"
+#define MDB_VERSION_DATE	"June 19, 2015"
 
 /** A stringifier for the version info */
 #define MDB_VERSTR(a,b,c,d)	"LMDB " #a "." #b "." #c ": (" d ")"

From 7d162bcf31a293242118a625a426becfb1974626 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 2 Jul 2015 18:03:36 +0100
Subject: [PATCH 082/504] CHANGES only belongs in mdb.RE

---
 libraries/liblmdb/CHANGES | 112 --------------------------------------
 1 file changed, 112 deletions(-)
 delete mode 100644 libraries/liblmdb/CHANGES

diff --git a/libraries/liblmdb/CHANGES b/libraries/liblmdb/CHANGES
deleted file mode 100644
index 34049ba7ac..0000000000
--- a/libraries/liblmdb/CHANGES
+++ /dev/null
@@ -1,112 +0,0 @@
-LMDB 0.9 Change Log
-
-LMDB 0.9.14 Release (2014/09/20)
-	Fix to support 64K page size (ITS#7713)
-	Fix to persist decreased as well as increased mapsizes (ITS#7789)
-	Fix cursor bug when deleting last node of a DUPSORT key
-	Fix mdb_env_info to return FIXEDMAP address
-	Fix ambiguous error code from writing to closed DBI (ITS#7825)
-	Fix mdb_copy copying past end of file (ITS#7886)
-	Fix cursor bugs from page_merge/rebalance
-	Fix to dirty fewer pages in deletes (mdb_page_loose())
-	Fix mdb_dbi_open creating subDBs (ITS#7917)
-	Fix mdb_cursor_get(_DUP) with single value (ITS#7913)
-	Fix Windows compat issues in mtests (ITS#7879)
-	Add compacting variant of mdb_copy
-	Add BigEndian integer key compare code
-	Add mdb_dump/mdb_load utilities
-
-LMDB 0.9.13 Release (2014/06/18)
-	Fix mdb_page_alloc unlimited overflow page search
-	Documentation
-		Re-fix MDB_CURRENT doc (ITS#7793)
-		Fix MDB_GET_MULTIPLE/MDB_NEXT_MULTIPLE doc
-
-LMDB 0.9.12 Release (2014/06/13)
-	Fix MDB_GET_BOTH regression (ITS#7875,#7681)
-	Fix MDB_MULTIPLE writing multiple keys (ITS#7834)
-	Fix mdb_rebalance (ITS#7829)
-	Fix mdb_page_split (ITS#7815)
-	Fix md_entries count (ITS#7861,#7828,#7793)
-	Fix MDB_CURRENT (ITS#7793)
-	Fix possible crash on Windows DLL detach
-	Misc code cleanup
-	Documentation
-		mdb_cursor_put: cursor moves on error (ITS#7771)
-
-
-LMDB 0.9.11 Release (2014/01/15)
-	Add mdb_env_set_assert() (ITS#7775)
-	Fix: invalidate txn on page allocation errors (ITS#7377)
-	Fix xcursor tracking in mdb_cursor_del0() (ITS#7771)
-	Fix corruption from deletes (ITS#7756)
-	Fix Windows/MSVC build issues
-	Raise safe limit of max MDB_MAXKEYSIZE
-	Misc code cleanup
-	Documentation
-		Remove spurious note about non-overlapping flags (ITS#7665)
-
-LMDB 0.9.10 Release (2013/11/12)
-	Add MDB_NOMEMINIT option
-	Fix mdb_page_split() again (ITS#7589)
-	Fix MDB_NORDAHEAD definition (ITS#7734)
-	Fix mdb_cursor_del() positioning (ITS#7733)
-	Partial fix for larger page sizes (ITS#7713)
-	Fix Windows64/MSVC build issues
-
-LMDB 0.9.9 Release (2013/10/24)
-	Add mdb_env_get_fd()
-	Add MDB_NORDAHEAD option
-	Add MDB_NOLOCK option
-	Avoid wasting space in mdb_page_split() (ITS#7589)
-	Fix mdb_page_merge() cursor fixup (ITS#7722)
-	Fix mdb_cursor_del() on last delete (ITS#7718)
-	Fix adding WRITEMAP on existing env (ITS#7715)
-	Fix nested txns (ITS#7515)
-	Fix mdb_env_copy() O_DIRECT bug (ITS#7682)
-	Fix mdb_cursor_set(SET_RANGE) return code (ITS#7681)
-	Fix mdb_rebalance() cursor fixup (ITS#7701)
-	Misc code cleanup
-	Documentation
-		Note that by default, readers need write access
-
-
-LMDB 0.9.8 Release (2013/09/09)
-	Allow mdb_env_set_mapsize() on an open environment
-	Fix mdb_dbi_flags() (ITS#7672)
-	Fix mdb_page_unspill() in nested txns
-	Fix mdb_cursor_get(CURRENT|NEXT) after a delete
-	Fix mdb_cursor_get(DUP) to always return key (ITS#7671)
-	Fix mdb_cursor_del() to always advance to next item (ITS#7670)
-	Fix mdb_cursor_set(SET_RANGE) for tree with single page (ITS#7681)
-	Fix mdb_env_copy() retry open if O_DIRECT fails (ITS#7682)
-	Tweak mdb_page_spill() to be less aggressive
-	Documentation
-		Update caveats since mdb_reader_check() added in 0.9.7
-
-LMDB 0.9.7 Release (2013/08/17)
-	Don't leave stale lockfile on failed RDONLY open (ITS#7664)
-	Fix mdb_page_split() ref beyond cursor depth
-	Fix read txn data race (ITS#7635)
-	Fix mdb_rebalance (ITS#7536, #7538)
-	Fix mdb_drop() (ITS#7561)
-	Misc DEBUG macro fixes
-	Add MDB_NOTLS envflag
-	Add mdb_env_copyfd()
-	Add mdb_txn_env() (ITS#7660)
-	Add mdb_dbi_flags() (ITS#7661)
-	Add mdb_env_get_maxkeysize()
-	Add mdb_env_reader_list()/mdb_env_reader_check()
-	Add mdb_page_spill/unspill, remove hard txn size limit
-	Use shorter names for semaphores (ITS#7615)
-	Build
-		Fix install target (ITS#7656)
-	Documentation
-		Misc updates for cursors, DB handles, data lifetime
-
-LMDB 0.9.6 Release (2013/02/25)
-	Many fixes/enhancements
-
-LMDB 0.9.5 Release (2012/11/30)
-	Renamed from libmdb to liblmdb
-	Many fixes/enhancements

From 8293b20cfed74b5a24ffef931ffa643663c02cd0 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Thu, 2 Jul 2015 21:06:27 +0200
Subject: [PATCH 083/504] Add MDB_USE_POSIX_MUTEX.

So far just to help aligning mdb.master and mdb.RE.
We'll make it an option later.
---
 libraries/liblmdb/mdb.c | 29 ++++++++++++++++++-----------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index cc0944d59b..5e2c24b289 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -128,9 +128,16 @@ union semun {
 	unsigned short *array;
 };
 #endif /* _SEM_SEMUN_UNDEFINED */
+#else
+#define MDB_USE_POSIX_MUTEX	1
 #endif /* MDB_USE_SYSV_SEM */
 #endif /* !_WIN32 */
 
+#if defined(_WIN32) + defined(MDB_USE_SYSV_SEM) \
+	+ defined(MDB_USE_POSIX_MUTEX) != 1
+# error "Ambiguous shared-lock implementation"
+#endif
+
 #ifdef USE_VALGRIND
 #include <valgrind/memcheck.h>
 #define VGMEMP_CREATE(h,r,z)    VALGRIND_CREATE_MEMPOOL(h,r,z)
@@ -206,10 +213,6 @@ union semun {
 	/** Features under development */
 #ifndef MDB_DEVEL
 #define MDB_DEVEL 0
-#endif
-
-#if defined(_WIN32) || defined(MDB_USE_SYSV_SEM) || defined(EOWNERDEAD)
-#define MDB_ROBUST_SUPPORTED	1
 #endif
 
 	/** Wrapper around __func__, which is a C99 feature */
@@ -228,8 +231,12 @@ union semun {
 #define MDB_OWNERDEAD	((int) WAIT_ABANDONED)
 #elif defined MDB_USE_SYSV_SEM
 #define MDB_OWNERDEAD	(MDB_LAST_ERRCODE + 11)
-#else
-#define MDB_OWNERDEAD	EOWNERDEAD
+#elif defined(MDB_USE_POSIX_MUTEX) && defined(EOWNERDEAD)
+#define MDB_OWNERDEAD	EOWNERDEAD	/**< #LOCK_MUTEX0() result if dead owner */
+#endif
+
+#ifdef MDB_OWNERDEAD
+#define MDB_ROBUST_SUPPORTED	1
 #endif
 
 #ifdef _WIN32
@@ -314,7 +321,7 @@ mdb_sem_wait(mdb_mutex_t *sem)
 
 #define mdb_mutex_consistent(mutex)	0
 
-#else
+#else	/* MDB_USE_POSIX_MUTEX: */
 	/** Pointer/HANDLE type of shared mutex/semaphore.
 	 */
 typedef pthread_mutex_t mdb_mutex_t;
@@ -1290,7 +1297,7 @@ static int	mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata,
 static int  mdb_env_read_header(MDB_env *env, MDB_meta *meta);
 static int  mdb_env_pick_meta(const MDB_env *env);
 static int  mdb_env_write_meta(MDB_txn *txn);
-#if !(defined(_WIN32) || defined(MDB_USE_SYSV_SEM)) /* Drop unused excl arg */
+#ifdef MDB_USE_POSIX_MUTEX /* Drop unused excl arg */
 # define mdb_env_close0(env, excl) mdb_env_close1(env)
 #endif
 static void mdb_env_close0(MDB_env *env, int excl);
@@ -4303,8 +4310,8 @@ mdb_env_excl_lock(MDB_env *env, int *excl)
 	if (!rc) {
 		*excl = 1;
 	} else
-# ifdef MDB_USE_SYSV_SEM
-	if (*excl < 0) /* always true when !MDB_USE_SYSV_SEM */
+# ifndef MDB_USE_POSIX_MUTEX
+	if (*excl < 0) /* always true when MDB_USE_POSIX_MUTEX */
 # endif
 	{
 		lock_info.l_type = F_RDLCK;
@@ -4558,7 +4565,7 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 		if (semctl(semid, 0, SETALL, semu) < 0)
 			goto fail_errno;
 		env->me_txns->mti_semid = semid;
-#else	/* MDB_USE_SYSV_SEM */
+#else	/* MDB_USE_POSIX_MUTEX: */
 		pthread_mutexattr_t mattr;
 
 		if ((rc = pthread_mutexattr_init(&mattr))

From 9574b65648dc20fb2e0f267a63068cc92e56bd2d Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Thu, 2 Jul 2015 21:06:35 +0200
Subject: [PATCH 084/504] Fix mdb_mutex_t, add mdb_mutexref_t, drop
 MDB_MUTEX().

Fixes the types of Windows HANDLEs, they became HANDLE*.
MDB_MUTEX() took a magic r/w arg which it's nice to get rid of.
---
 libraries/liblmdb/mdb.c | 77 +++++++++++++++++++++--------------------
 1 file changed, 40 insertions(+), 37 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 5e2c24b289..0fc3ab7242 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -246,7 +246,7 @@ union semun {
 #define pthread_t	HANDLE
 #define pthread_mutex_t	HANDLE
 #define pthread_cond_t	HANDLE
-typedef HANDLE mdb_mutex_t;
+typedef HANDLE mdb_mutex_t, mdb_mutexref_t;
 #define pthread_key_t	DWORD
 #define pthread_self()	GetCurrentThreadId()
 #define pthread_key_create(x,y)	\
@@ -260,7 +260,6 @@ typedef HANDLE mdb_mutex_t;
 #define pthread_cond_wait(cond,mutex)	do{SignalObjectAndWait(*mutex, *cond, INFINITE, FALSE); WaitForSingleObject(*mutex, INFINITE);}while(0)
 #define THREAD_CREATE(thr,start,arg)	thr=CreateThread(NULL,0,start,arg,0,NULL)
 #define THREAD_FINISH(thr)	WaitForSingleObject(thr, INFINITE)
-#define MDB_MUTEX(env, rw)		((env)->me_##rw##mutex)
 #define LOCK_MUTEX0(mutex)		WaitForSingleObject(mutex, INFINITE)
 #define UNLOCK_MUTEX(mutex)		ReleaseMutex(mutex)
 #define mdb_mutex_consistent(mutex)	0
@@ -292,9 +291,8 @@ typedef struct mdb_mutex {
 	int semid;
 	int semnum;
 	int *locked;
-} mdb_mutex_t;
+} mdb_mutex_t[1], *mdb_mutexref_t;
 
-#define MDB_MUTEX(env, rw)		(&(env)->me_##rw##mutex)
 #define LOCK_MUTEX0(mutex)		mdb_sem_wait(mutex)
 #define UNLOCK_MUTEX(mutex)		do { \
 	struct sembuf sb = { 0, 1, SEM_UNDO }; \
@@ -304,7 +302,7 @@ typedef struct mdb_mutex {
 } while(0)
 
 static int
-mdb_sem_wait(mdb_mutex_t *sem)
+mdb_sem_wait(mdb_mutexref_t sem)
 {
 	int rc, *locked = sem->locked;
 	struct sembuf sb = { 0, -1, SEM_UNDO };
@@ -322,12 +320,15 @@ mdb_sem_wait(mdb_mutex_t *sem)
 #define mdb_mutex_consistent(mutex)	0
 
 #else	/* MDB_USE_POSIX_MUTEX: */
-	/** Pointer/HANDLE type of shared mutex/semaphore.
+	/** Shared mutex/semaphore as it is stored (mdb_mutex_t), and as
+	 *	local variables keep it (mdb_mutexref_t).
+	 *
+	 *	An mdb_mutex_t can be assigned to an mdb_mutexref_t.  They can
+	 *	be the same, or an array[size 1] and a pointer.
+	 *	@{
 	 */
-typedef pthread_mutex_t mdb_mutex_t;
-	/** Mutex for the reader table (rw = r) or write transaction (rw = w).
-	 */
-#define MDB_MUTEX(env, rw)	(&(env)->me_txns->mti_##rw##mutex)
+typedef pthread_mutex_t mdb_mutex_t[1], *mdb_mutexref_t;
+	/*	@} */
 	/** Lock the reader or writer mutex.
 	 *	Returns 0 or a code to give #mdb_mutex_failed(), as in #LOCK_MUTEX().
 	 */
@@ -386,7 +387,7 @@ typedef pthread_mutex_t mdb_mutex_t;
 #define LOCK_MUTEX(rc, env, mutex) \
 	(((rc) = LOCK_MUTEX0(mutex)) && \
 	 ((rc) = mdb_mutex_failed(env, mutex, rc)))
-static int mdb_mutex_failed(MDB_env *env, mdb_mutex_t *mutex, int rc);
+static int mdb_mutex_failed(MDB_env *env, mdb_mutexref_t mutex, int rc);
 #else
 #define LOCK_MUTEX(rc, env, mutex) ((rc) = LOCK_MUTEX0(mutex))
 #define mdb_mutex_failed(env, mutex, rc) (rc)
@@ -711,9 +712,9 @@ typedef struct MDB_txbody {
 	int		mtb_rlocked;
 #else
 		/** Mutex protecting access to this table.
-		 *	This is the #MDB_MUTEX(env,r) reader table lock.
+		 *	This is the reader table lock used with LOCK_MUTEX().
 		 */
-	pthread_mutex_t	mtb_rmutex;
+	mdb_mutex_t	mtb_rmutex;
 #endif
 		/**	The ID of the last transaction committed to the database.
 		 *	This is recorded here only for convenience; the value can always
@@ -751,7 +752,7 @@ typedef struct MDB_txninfo {
 		int mt2_wlocked;
 #define mti_wlocked	mt2.mt2_wlocked
 #else
-		pthread_mutex_t	mt2_wmutex;
+		mdb_mutex_t	mt2_wmutex;
 #define mti_wmutex	mt2.mt2_wmutex
 #endif
 		char pad[(MNAME_LEN+CACHELINE-1) & ~(CACHELINE-1)];
@@ -1242,8 +1243,10 @@ struct MDB_env {
 #ifdef _WIN32
 	int		me_pidquery;		/**< Used in OpenProcess */
 #endif
-#if defined(_WIN32) || defined(MDB_USE_SYSV_SEM)
-	/* Windows mutexes/SysV semaphores do not reside in shared mem */
+#ifdef MDB_USE_POSIX_MUTEX	/* Posix mutexes reside in shared mem */
+#	define		me_rmutex	me_txns->mti_rmutex /**< Shared reader lock */
+#	define		me_wmutex	me_txns->mti_wmutex /**< Shared writer lock */
+#else
 	mdb_mutex_t	me_rmutex;
 	mdb_mutex_t	me_wmutex;
 #endif
@@ -2611,7 +2614,7 @@ mdb_txn_renew0(MDB_txn *txn)
 			} else {
 				MDB_PID_T pid = env->me_pid;
 				MDB_THR_T tid = pthread_self();
-				mdb_mutex_t *rmutex = MDB_MUTEX(env, r);
+				mdb_mutexref_t rmutex = env->me_rmutex;
 
 				if (!env->me_live_reader) {
 					rc = mdb_reader_pid(env, Pidset, pid);
@@ -2663,7 +2666,7 @@ mdb_txn_renew0(MDB_txn *txn)
 	} else {
 		/* Not yet touching txn == env->me_txn0, it may be active */
 		if (ti) {
-			if (LOCK_MUTEX(rc, env, MDB_MUTEX(env, w)))
+			if (LOCK_MUTEX(rc, env, env->me_wmutex))
 				return rc;
 			txn->mt_txnid = ti->mti_txnid;
 			meta = env->me_metas[txn->mt_txnid & 1];
@@ -2941,7 +2944,7 @@ mdb_txn_reset0(MDB_txn *txn, const char *act)
 			env->me_txn = NULL;
 			/* The writer mutex was locked in mdb_txn_begin. */
 			if (env->me_txns)
-				UNLOCK_MUTEX(MDB_MUTEX(env, w));
+				UNLOCK_MUTEX(env->me_wmutex);
 		} else {
 			txn->mt_parent->mt_child = NULL;
 			env->me_pgstate = ((MDB_ntxn *)txn)->mnt_pgstate;
@@ -3543,7 +3546,7 @@ done:
 	mdb_dbis_update(txn, 1);
 
 	if (env->me_txns)
-		UNLOCK_MUTEX(MDB_MUTEX(env, w));
+		UNLOCK_MUTEX(env->me_wmutex);
 	if (txn != env->me_txn0)
 		free(txn);
 
@@ -3838,8 +3841,8 @@ mdb_env_create(MDB_env **env)
 	e->me_lfd = INVALID_HANDLE_VALUE;
 	e->me_mfd = INVALID_HANDLE_VALUE;
 #ifdef MDB_USE_SYSV_SEM
-	e->me_rmutex.semid = -1;
-	e->me_wmutex.semid = -1;
+	e->me_rmutex->semid = -1;
+	e->me_wmutex->semid = -1;
 #endif
 	e->me_pid = getpid();
 	GET_PAGESIZE(e->me_os_psize);
@@ -4573,8 +4576,8 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 #ifdef MDB_ROBUST_SUPPORTED
 			|| (rc = pthread_mutexattr_setrobust(&mattr, PTHREAD_MUTEX_ROBUST))
 #endif
-			|| (rc = pthread_mutex_init(&env->me_txns->mti_rmutex, &mattr))
-			|| (rc = pthread_mutex_init(&env->me_txns->mti_wmutex, &mattr)))
+			|| (rc = pthread_mutex_init(env->me_txns->mti_rmutex, &mattr))
+			|| (rc = pthread_mutex_init(env->me_txns->mti_wmutex, &mattr)))
 			goto fail;
 		pthread_mutexattr_destroy(&mattr);
 #endif	/* _WIN32 || MDB_USE_SYSV_SEM */
@@ -4620,12 +4623,12 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 #endif
 	}
 #ifdef MDB_USE_SYSV_SEM
-	env->me_rmutex.semid = semid;
-	env->me_wmutex.semid = semid;
-	env->me_rmutex.semnum = 0;
-	env->me_wmutex.semnum = 1;
-	env->me_rmutex.locked = &env->me_txns->mti_rlocked;
-	env->me_wmutex.locked = &env->me_txns->mti_wlocked;
+	env->me_rmutex->semid = semid;
+	env->me_wmutex->semid = semid;
+	env->me_rmutex->semnum = 0;
+	env->me_wmutex->semnum = 1;
+	env->me_rmutex->locked = &env->me_txns->mti_rlocked;
+	env->me_wmutex->locked = &env->me_txns->mti_wlocked;
 #endif
 
 	return MDB_SUCCESS;
@@ -4862,14 +4865,14 @@ mdb_env_close0(MDB_env *env, int excl)
 		 * the last handle closes.
 		 */
 #elif defined(MDB_USE_SYSV_SEM)
-		if (env->me_rmutex.semid != -1) {
+		if (env->me_rmutex->semid != -1) {
 			/* If we have the filelock:  If we are the
 			 * only remaining user, clean up semaphores.
 			 */
 			if (excl == 0)
 				mdb_env_excl_lock(env, &excl);
 			if (excl > 0)
-				semctl(env->me_rmutex.semid, 0, IPC_RMID);
+				semctl(env->me_rmutex->semid, 0, IPC_RMID);
 		}
 #endif
 		munmap((void *)env->me_txns, (env->me_maxreaders-1)*sizeof(MDB_reader)+sizeof(MDB_txninfo));
@@ -8908,7 +8911,7 @@ static int ESECT
 mdb_env_copyfd0(MDB_env *env, HANDLE fd)
 {
 	MDB_txn *txn = NULL;
-	mdb_mutex_t *wmutex = NULL;
+	mdb_mutexref_t wmutex = NULL;
 	int rc;
 	size_t wsize;
 	char *ptr;
@@ -8933,7 +8936,7 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd)
 		mdb_txn_reset0(txn, "reset-stage1");
 
 		/* Temporarily block writers until we snapshot the meta pages */
-		wmutex = MDB_MUTEX(env, w);
+		wmutex = env->me_wmutex;
 		if (LOCK_MUTEX(rc, env, wmutex))
 			goto leave;
 
@@ -9647,7 +9650,7 @@ mdb_reader_check(MDB_env *env, int *dead)
 /** As #mdb_reader_check(). rlocked = <caller locked the reader mutex>. */
 static int mdb_reader_check0(MDB_env *env, int rlocked, int *dead)
 {
-	mdb_mutex_t *rmutex = rlocked ? NULL : MDB_MUTEX(env, r);
+	mdb_mutexref_t rmutex = rlocked ? NULL : env->me_rmutex;
 	unsigned int i, j, rdrs;
 	MDB_reader *mr;
 	MDB_PID_T *pids, pid;
@@ -9704,14 +9707,14 @@ static int mdb_reader_check0(MDB_env *env, int rlocked, int *dead)
  * @param[in] rc	LOCK_MUTEX0() error (nonzero)
  * @return 0 on success with the mutex locked, or an error code on failure.
  */
-static int mdb_mutex_failed(MDB_env *env, mdb_mutex_t *mutex, int rc)
+static int mdb_mutex_failed(MDB_env *env, mdb_mutexref_t mutex, int rc)
 {
 	int toggle, rlocked, rc2;
 
 	if (rc == MDB_OWNERDEAD) {
 		/* We own the mutex. Clean up after dead previous owner. */
 		rc = MDB_SUCCESS;
-		rlocked = (mutex == MDB_MUTEX(env, r));
+		rlocked = (mutex == env->me_rmutex);
 		if (!rlocked) {
 			/* Keep mti_txnid updated, otherwise next writer can
 			 * overwrite data which latest meta page refers to.

From 943b23f01141ec0e9f85fa166b7867d63afcc5a9 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Thu, 2 Jul 2015 21:14:56 +0200
Subject: [PATCH 085/504] Restore support for Posix semaphores

---
 libraries/liblmdb/Makefile |  2 +-
 libraries/liblmdb/lmdb.h   |  3 +-
 libraries/liblmdb/mdb.c    | 96 ++++++++++++++++++++++++++++++++++----
 3 files changed, 89 insertions(+), 12 deletions(-)

diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile
index 2d0983eff0..a8ea9706fa 100644
--- a/libraries/liblmdb/Makefile
+++ b/libraries/liblmdb/Makefile
@@ -8,7 +8,7 @@
 # platforms; you should not need to change any of these.
 # Read their descriptions in mdb.c if you do:
 #
-# - MDB_USE_POSIX_SEM
+# - MDB_USE_POSIX_MUTEX, MDB_USE_POSIX_SEM, MDB_USE_SYSV_SEM
 # - MDB_DSYNC
 # - MDB_FDATASYNC
 # - MDB_FDATASYNC_WORKS
diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index e9a01a5cb9..deb6812578 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -57,7 +57,8 @@
  *	  Otherwise just make all programs using the database close it;
  *	  the lockfile is always reset on first open of the environment.
  *
- *	- On BSD systems or others configured with MDB_USE_SYSV_SEM,
+ *	- On BSD systems or others configured with MDB_USE_SYSV_SEM or
+ *	  MDB_USE_POSIX_SEM,
  *	  startup can fail due to semaphores owned by another userid.
  *
  *	  Fix: Open and close the database as the user which owns the
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 0fc3ab7242..2d2486d02b 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -110,7 +110,9 @@ extern int cacheflush(char *addr, int nbytes, int cache);
 #endif
 
 #if defined(__APPLE__) || defined (BSD)
+# if !(defined(MDB_USE_POSIX_MUTEX) || defined(MDB_USE_POSIX_SEM))
 # define MDB_USE_SYSV_SEM	1
+# endif
 # define MDB_FDATASYNC		fsync
 #elif defined(ANDROID)
 # define MDB_FDATASYNC		fsync
@@ -118,7 +120,10 @@ extern int cacheflush(char *addr, int nbytes, int cache);
 
 #ifndef _WIN32
 #include <pthread.h>
-#ifdef MDB_USE_SYSV_SEM
+#ifdef MDB_USE_POSIX_SEM
+# define MDB_USE_HASH		1
+#include <semaphore.h>
+#elif defined(MDB_USE_SYSV_SEM)
 #include <sys/ipc.h>
 #include <sys/sem.h>
 #ifdef _SEM_SEMUN_UNDEFINED
@@ -130,10 +135,10 @@ union semun {
 #endif /* _SEM_SEMUN_UNDEFINED */
 #else
 #define MDB_USE_POSIX_MUTEX	1
-#endif /* MDB_USE_SYSV_SEM */
+#endif /* MDB_USE_POSIX_SEM */
 #endif /* !_WIN32 */
 
-#if defined(_WIN32) + defined(MDB_USE_SYSV_SEM) \
+#if defined(_WIN32) + defined(MDB_USE_POSIX_SEM) + defined(MDB_USE_SYSV_SEM) \
 	+ defined(MDB_USE_POSIX_MUTEX) != 1
 # error "Ambiguous shared-lock implementation"
 #endif
@@ -285,7 +290,21 @@ typedef HANDLE mdb_mutex_t, mdb_mutexref_t;
 	/** For MDB_LOCK_FORMAT: True if readers take a pid lock in the lockfile */
 #define MDB_PIDLOCK			1
 
-#ifdef MDB_USE_SYSV_SEM
+#ifdef MDB_USE_POSIX_SEM
+
+typedef sem_t *mdb_mutex_t, *mdb_mutexref_t;
+#define LOCK_MUTEX0(mutex)		mdb_sem_wait(mutex)
+#define UNLOCK_MUTEX(mutex)		sem_post(mutex)
+
+static int
+mdb_sem_wait(sem_t *sem)
+{
+   int rc;
+   while ((rc = sem_wait(sem)) && (rc = errno) == EINTR) ;
+   return rc;
+}
+
+#elif defined MDB_USE_SYSV_SEM
 
 typedef struct mdb_mutex {
 	int semid;
@@ -339,7 +358,7 @@ typedef pthread_mutex_t mdb_mutex_t[1], *mdb_mutexref_t;
 	/** Mark mutex-protected data as repaired, after death of previous owner.
 	 */
 #define mdb_mutex_consistent(mutex)	pthread_mutex_consistent(mutex)
-#endif	/* MDB_USE_SYSV_SEM */
+#endif	/* MDB_USE_POSIX_SEM || MDB_USE_SYSV_SEM */
 
 	/** Get the error code for the last failed system function.
 	 */
@@ -364,7 +383,7 @@ typedef pthread_mutex_t mdb_mutex_t[1], *mdb_mutexref_t;
 #define	GET_PAGESIZE(x)	((x) = sysconf(_SC_PAGE_SIZE))
 #endif
 
-#if defined(_WIN32)
+#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM)
 #define MNAME_LEN	32
 #elif defined(MDB_USE_SYSV_SEM)
 #define MNAME_LEN	(sizeof(int))
@@ -705,7 +724,7 @@ typedef struct MDB_txbody {
 	uint32_t	mtb_magic;
 		/** Format of this lock file. Must be set to #MDB_LOCK_FORMAT. */
 	uint32_t	mtb_format;
-#if defined(_WIN32)
+#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM)
 	char	mtb_rmname[MNAME_LEN];
 #elif defined(MDB_USE_SYSV_SEM)
 	int 	mtb_semid;
@@ -745,7 +764,7 @@ typedef struct MDB_txninfo {
 		char pad[(sizeof(MDB_txbody)+CACHELINE-1) & ~(CACHELINE-1)];
 	} mt1;
 	union {
-#if defined(_WIN32)
+#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM)
 		char mt2_wmname[MNAME_LEN];
 #define	mti_wmname	mt2.mt2_wmname
 #elif defined MDB_USE_SYSV_SEM
@@ -3840,7 +3859,10 @@ mdb_env_create(MDB_env **env)
 	e->me_fd = INVALID_HANDLE_VALUE;
 	e->me_lfd = INVALID_HANDLE_VALUE;
 	e->me_mfd = INVALID_HANDLE_VALUE;
-#ifdef MDB_USE_SYSV_SEM
+#ifdef MDB_USE_POSIX_SEM
+	e->me_rmutex = SEM_FAILED;
+	e->me_wmutex = SEM_FAILED;
+#elif defined MDB_USE_SYSV_SEM
 	e->me_rmutex->semid = -1;
 	e->me_wmutex->semid = -1;
 #endif
@@ -4556,6 +4578,40 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 		if (!env->me_rmutex) goto fail_errno;
 		env->me_wmutex = CreateMutex(&mdb_all_sa, FALSE, env->me_txns->mti_wmname);
 		if (!env->me_wmutex) goto fail_errno;
+#elif defined(MDB_USE_POSIX_SEM)
+		struct stat stbuf;
+		struct {
+			dev_t dev;
+			ino_t ino;
+		} idbuf;
+		MDB_val val;
+		char encbuf[11];
+
+#if defined(__NetBSD__)
+#define	MDB_SHORT_SEMNAMES	1	/* limited to 14 chars */
+#endif
+		if (fstat(env->me_lfd, &stbuf)) goto fail_errno;
+		idbuf.dev = stbuf.st_dev;
+		idbuf.ino = stbuf.st_ino;
+		val.mv_data = &idbuf;
+		val.mv_size = sizeof(idbuf);
+		mdb_hash_enc(&val, encbuf);
+#ifdef MDB_SHORT_SEMNAMES
+		encbuf[9] = '\0';	/* drop name from 15 chars to 14 chars */
+#endif
+		sprintf(env->me_txns->mti_rmname, "/MDBr%s", encbuf);
+		sprintf(env->me_txns->mti_wmname, "/MDBw%s", encbuf);
+		/* Clean up after a previous run, if needed:  Try to
+		 * remove both semaphores before doing anything else.
+		 */
+		sem_unlink(env->me_txns->mti_rmname);
+		sem_unlink(env->me_txns->mti_wmname);
+		env->me_rmutex = sem_open(env->me_txns->mti_rmname,
+			O_CREAT|O_EXCL, mode, 1);
+		if (env->me_rmutex == SEM_FAILED) goto fail_errno;
+		env->me_wmutex = sem_open(env->me_txns->mti_wmname,
+			O_CREAT|O_EXCL, mode, 1);
+		if (env->me_wmutex == SEM_FAILED) goto fail_errno;
 #elif defined(MDB_USE_SYSV_SEM)
 		unsigned short vals[2] = {1, 1};
 		key_t key = ftok(lpath, 'M');
@@ -4580,7 +4636,7 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 			|| (rc = pthread_mutex_init(env->me_txns->mti_wmutex, &mattr)))
 			goto fail;
 		pthread_mutexattr_destroy(&mattr);
-#endif	/* _WIN32 || MDB_USE_SYSV_SEM */
+#endif	/* _WIN32 || ... */
 
 		env->me_txns->mti_magic = MDB_MAGIC;
 		env->me_txns->mti_format = MDB_LOCK_FORMAT;
@@ -4611,6 +4667,11 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 		if (!env->me_rmutex) goto fail_errno;
 		env->me_wmutex = OpenMutex(SYNCHRONIZE, FALSE, env->me_txns->mti_wmname);
 		if (!env->me_wmutex) goto fail_errno;
+#elif defined(MDB_USE_POSIX_SEM)
+		env->me_rmutex = sem_open(env->me_txns->mti_rmname, 0);
+		if (env->me_rmutex == SEM_FAILED) goto fail_errno;
+		env->me_wmutex = sem_open(env->me_txns->mti_wmname, 0);
+		if (env->me_wmutex == SEM_FAILED) goto fail_errno;
 #elif defined(MDB_USE_SYSV_SEM)
 		semid = env->me_txns->mti_semid;
 		semu.buf = &buf;
@@ -4864,6 +4925,21 @@ mdb_env_close0(MDB_env *env, int excl)
 		/* Windows automatically destroys the mutexes when
 		 * the last handle closes.
 		 */
+#elif defined(MDB_USE_POSIX_SEM)
+		if (env->me_rmutex != SEM_FAILED) {
+			sem_close(env->me_rmutex);
+			if (env->me_wmutex != SEM_FAILED)
+				sem_close(env->me_wmutex);
+			/* If we have the filelock:  If we are the
+			 * only remaining user, clean up semaphores.
+			 */
+			if (excl == 0)
+				mdb_env_excl_lock(env, &excl);
+			if (excl > 0) {
+				sem_unlink(env->me_txns->mti_rmname);
+				sem_unlink(env->me_txns->mti_wmname);
+			}
+		}
 #elif defined(MDB_USE_SYSV_SEM)
 		if (env->me_rmutex->semid != -1) {
 			/* If we have the filelock:  If we are the

From a937740aa0c47dc7a1a5e9be42dcea2dd3c81683 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Fri, 3 Jul 2015 22:38:50 +0200
Subject: [PATCH 086/504] ITS#7969 Wrap unportable __sync_synchronize in #if

---
 libraries/liblmdb/mdb.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 2d2486d02b..ae82bb2fa4 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -3744,7 +3744,8 @@ mdb_env_write_meta(MDB_txn *txn)
 		mp->mm_dbs[0] = txn->mt_dbs[0];
 		mp->mm_dbs[1] = txn->mt_dbs[1];
 		mp->mm_last_pg = txn->mt_next_pgno - 1;
-#if !(defined(_MSC_VER) || defined(__i386__) || defined(__x86_64__))
+#if (__GNUC__ * 100 + __GNUC_MINOR__ >= 404) && /* TODO: portability */	\
+	!(defined(__i386__) || defined(__x86_64__))
 		/* LY: issue a memory barrier, if not x86. ITS#7969 */
 		__sync_synchronize();
 #endif

From 11536bea2533db3c1f1fcf9b0864ecaee11da851 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 4 Jul 2015 13:42:41 +0200
Subject: [PATCH 087/504] mdb_env_get_flags(): Hide internal flags

---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index ae82bb2fa4..82701000a1 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -9182,7 +9182,7 @@ mdb_env_get_flags(MDB_env *env, unsigned int *arg)
 	if (!env || !arg)
 		return EINVAL;
 
-	*arg = env->me_flags;
+	*arg = env->me_flags & (CHANGEABLE|CHANGELESS);
 	return MDB_SUCCESS;
 }
 

From f702e33b6cdf750a0affab64144379138bcf997c Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 4 Jul 2015 13:48:01 +0200
Subject: [PATCH 088/504] Use mdb_cmp_long() for FREE_DBI

---
 libraries/liblmdb/mdb.c | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 82701000a1..7cf52943a2 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -4769,6 +4769,7 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 		rc = ENOMEM;
 		goto leave;
 	}
+	env->me_dbxs[FREE_DBI].md_cmp = mdb_cmp_long; /* aligned MDB_INTEGERKEY */
 
 	/* For RDONLY, get lockfile after we know datafile exists */
 	if (!(flags & (MDB_RDONLY|MDB_NOLOCK))) {
@@ -9316,10 +9317,6 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db
 	unsigned int unused = 0, seq;
 	size_t len;
 
-	if (txn->mt_dbxs[FREE_DBI].md_cmp == NULL) {
-		mdb_default_cmp(txn, FREE_DBI);
-	}
-
 	if ((flags & VALID_FLAGS) != flags)
 		return EINVAL;
 	if (txn->mt_flags & MDB_TXN_ERROR)

From b6ac0cd90ac3bdd1c464dc3a1d304772769bb75b Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 4 Jul 2015 13:48:02 +0200
Subject: [PATCH 089/504] mdb_drop0(): Omit scanning DUPSORT sub-DB leaves

---
 libraries/liblmdb/mdb.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 7cf52943a2..1de63e2668 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -9470,8 +9470,10 @@ mdb_drop0(MDB_cursor *mc, int subs)
 		MDB_cursor mx;
 		unsigned int i;
 
-		/* LEAF2 pages have no nodes, cannot have sub-DBs */
-		if (IS_LEAF2(mc->mc_pg[mc->mc_top]))
+		/* DUPSORT sub-DBs have no ovpages/DBs. Omit scanning leaves.
+		 * This also avoids any P_LEAF2 pages, which have no nodes.
+		 */
+		if (mc->mc_flags & C_SUB)
 			mdb_cursor_pop(mc);
 
 		mdb_cursor_copy(mc, &mx);

From 530587158528f87d65b097a90885c0811684fdb7 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 4 Jul 2015 13:48:03 +0200
Subject: [PATCH 090/504] mdb_txn_begin() cleanup

---
 libraries/liblmdb/mdb.c | 33 ++++++++++++---------------------
 1 file changed, 12 insertions(+), 21 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 1de63e2668..b5db13bac5 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2779,7 +2779,6 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 	if (env->me_flags & MDB_RDONLY & ~flags) /* write txn in RDONLY env */
 		return EACCES;
 
-	size = tsize = sizeof(MDB_txn);
 	if (parent) {
 		/* Nested transactions: Max 1 child, write txns only, no writemap */
 		flags |= parent->mt_flags;
@@ -2789,40 +2788,31 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 			return (parent->mt_flags & MDB_TXN_RDONLY) ? EINVAL : MDB_BAD_TXN;
 		}
 		/* Child txns save MDB_pgstate and use own copy of cursors */
-		size = tsize = sizeof(MDB_ntxn);
-		size += env->me_maxdbs * sizeof(MDB_cursor *);
-	} else if (!(flags & MDB_RDONLY)) {
+		size = env->me_maxdbs * (sizeof(MDB_db)+sizeof(MDB_cursor *)+1);
+		size += tsize = sizeof(MDB_ntxn);
+	} else if (flags & MDB_RDONLY) {
+		size = env->me_maxdbs * (sizeof(MDB_db)+1);
+		size += tsize = sizeof(MDB_txn);
+	} else {
 		/* Reuse preallocated write txn. However, do not touch it until
 		 * mdb_txn_renew0() succeeds, since it currently may be active.
 		 */
 		txn = env->me_txn0;
 		goto renew;
 	}
-	size += env->me_maxdbs * (sizeof(MDB_db)+1);
-
 	if ((txn = calloc(1, size)) == NULL) {
 		DPRINTF(("calloc: %s", strerror(errno)));
 		return ENOMEM;
 	}
 	txn->mt_dbs = (MDB_db *) ((char *)txn + tsize);
-	if (flags & MDB_RDONLY) {
-		txn->mt_dbflags = (unsigned char *)(txn->mt_dbs + env->me_maxdbs);
-		txn->mt_dbiseqs = env->me_dbiseqs;
-	} else {
-		txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs);
-		if (parent) {
-			txn->mt_dbiseqs = parent->mt_dbiseqs;
-			txn->mt_dbflags = (unsigned char *)(txn->mt_cursors + env->me_maxdbs);
-		} else {
-			txn->mt_dbiseqs = (unsigned int *)(txn->mt_cursors + env->me_maxdbs);
-			txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs);
-		}
-	}
+	txn->mt_dbflags = (unsigned char *)txn + size - env->me_maxdbs;
 	txn->mt_flags = flags;
 	txn->mt_env = env;
 
 	if (parent) {
 		unsigned int i;
+		txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs);
+		txn->mt_dbiseqs = parent->mt_dbiseqs;
 		txn->mt_u.dirty_list = malloc(sizeof(MDB_ID2)*MDB_IDL_UM_SIZE);
 		if (!txn->mt_u.dirty_list ||
 			!(txn->mt_free_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX)))
@@ -2859,7 +2849,8 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 			rc = mdb_cursor_shadow(parent, txn);
 		if (rc)
 			mdb_txn_reset0(txn, "beginchild-fail");
-	} else {
+	} else { /* MDB_RDONLY */
+		txn->mt_dbiseqs = env->me_dbiseqs;
 renew:
 		rc = mdb_txn_renew0(txn);
 	}
@@ -2867,7 +2858,7 @@ renew:
 		if (txn != env->me_txn0)
 			free(txn);
 	} else {
-		txn->mt_flags |= flags;	/* for txn==me_txn0, no effect otherwise */
+		txn->mt_flags |= flags;	/* could not change txn=me_txn0 earlier */
 		*ret = txn;
 		DPRINTF(("begin txn %"Z"u%c %p on mdbenv %p, root page %"Z"u",
 			txn->mt_txnid, (flags & MDB_RDONLY) ? 'r' : 'w',

From 084d412f9b6d89806cd8dbbaa0b2d3a59470dc45 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 4 Jul 2015 13:48:04 +0200
Subject: [PATCH 091/504] Simpler mdb_txn_commit().

mt_env is always set.
Commit(mt_child) resets mt_child, so parent need not.
---
 libraries/liblmdb/mdb.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index b5db13bac5..00a6d07dc9 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -3349,12 +3349,11 @@ mdb_txn_commit(MDB_txn *txn)
 	unsigned int i;
 	MDB_env	*env;
 
-	if (txn == NULL || txn->mt_env == NULL)
+	if (txn == NULL)
 		return EINVAL;
 
 	if (txn->mt_child) {
 		rc = mdb_txn_commit(txn->mt_child);
-		txn->mt_child = NULL;
 		if (rc)
 			goto fail;
 	}

From be0cdc1c9dbe9105a8066087e8b3f2249b111a11 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 4 Jul 2015 13:48:05 +0200
Subject: [PATCH 092/504] Simpler mdb_node_shrink()

---
 libraries/liblmdb/mdb.c | 37 +++++++++++++++----------------------
 1 file changed, 15 insertions(+), 22 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 00a6d07dc9..f69a3a268e 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7183,45 +7183,38 @@ mdb_node_shrink(MDB_page *mp, indx_t indx)
 	MDB_node *node;
 	MDB_page *sp, *xp;
 	char *base;
-	int nsize, delta;
-	indx_t		 i, numkeys, ptr;
+	indx_t delta, nsize, len, ptr;
+	int i;
 
 	node = NODEPTR(mp, indx);
 	sp = (MDB_page *)NODEDATA(node);
 	delta = SIZELEFT(sp);
-	xp = (MDB_page *)((char *)sp + delta);
+	nsize = NODEDSZ(node) - delta;
 
-	/* shift subpage upward */
+	/* Prepare to shift upward, set len = length(subpage part to shift) */
 	if (IS_LEAF2(sp)) {
-		nsize = NUMKEYS(sp) * sp->mp_pad;
+		len = nsize;
 		if (nsize & 1)
 			return;		/* do not make the node uneven-sized */
-		memmove(METADATA(xp), METADATA(sp), nsize);
 	} else {
-		int i;
-		numkeys = NUMKEYS(sp);
-		for (i=numkeys-1; i>=0; i--)
+		xp = (MDB_page *)((char *)sp + delta); /* destination subpage */
+		for (i = NUMKEYS(sp); --i >= 0; )
 			xp->mp_ptrs[i] = sp->mp_ptrs[i] - delta;
+		len = PAGEHDRSZ;
 	}
-	xp->mp_upper = sp->mp_lower;
-	xp->mp_lower = sp->mp_lower;
-	xp->mp_flags = sp->mp_flags;
-	xp->mp_pad = sp->mp_pad;
-	COPY_PGNO(xp->mp_pgno, mp->mp_pgno);
-
-	nsize = NODEDSZ(node) - delta;
+	sp->mp_upper = sp->mp_lower;
+	COPY_PGNO(sp->mp_pgno, mp->mp_pgno);
 	SETDSZ(node, nsize);
 
-	/* shift lower nodes upward */
+	/* Shift <lower nodes...initial part of subpage> upward */
+	base = (char *)mp + mp->mp_upper + PAGEBASE;
+	memmove(base + delta, base, (char *)sp + len - base);
+
 	ptr = mp->mp_ptrs[indx];
-	numkeys = NUMKEYS(mp);
-	for (i = 0; i < numkeys; i++) {
+	for (i = NUMKEYS(mp); --i >= 0; ) {
 		if (mp->mp_ptrs[i] <= ptr)
 			mp->mp_ptrs[i] += delta;
 	}
-
-	base = (char *)mp + mp->mp_upper + PAGEBASE;
-	memmove(base + delta, base, ptr - mp->mp_upper + NODESIZE + NODEKSZ(node));
 	mp->mp_upper += delta;
 }
 

From 4033f31acbed3d136a35103c9c7f514816b19681 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 4 Jul 2015 13:48:06 +0200
Subject: [PATCH 093/504] Simpler mdb_drop().

MDB_DBI_CHANGED(,MAIN_DBI) is never true.
---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index f69a3a268e..bbfdd94d4b 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -9536,7 +9536,7 @@ int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del)
 	if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY))
 		return EACCES;
 
-	if (dbi > MAIN_DBI && TXN_DBI_CHANGED(txn, dbi))
+	if (TXN_DBI_CHANGED(txn, dbi))
 		return MDB_BAD_DBI;
 
 	rc = mdb_cursor_open(txn, dbi, &mc);

From 5cd5e928c1bb140487670fe18a4637e787f2f047 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 4 Jul 2015 13:48:07 +0200
Subject: [PATCH 094/504] Simpler flag/DBI checks and MDB_DEBUG

---
 libraries/liblmdb/mdb.c | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index bbfdd94d4b..c8f26bf21b 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5191,15 +5191,12 @@ static void
 mdb_cursor_pop(MDB_cursor *mc)
 {
 	if (mc->mc_snum) {
-#if MDB_DEBUG
-		MDB_page	*top = mc->mc_pg[mc->mc_top];
-#endif
+		DPRINTF(("popping page %"Z"u off db %d cursor %p",
+			mc->mc_pg[mc->mc_top]->mp_pgno, DDBI(mc), (void *) mc));
+
 		mc->mc_snum--;
 		if (mc->mc_snum)
 			mc->mc_top--;
-
-		DPRINTF(("popped page %"Z"u off db %d cursor %p", top->mp_pgno,
-			DDBI(mc), (void *) mc));
 	}
 }
 
@@ -8581,7 +8578,7 @@ mdb_put(MDB_txn *txn, MDB_dbi dbi,
 	if (!key || !data || dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi))
 		return EINVAL;
 
-	if ((flags & (MDB_NOOVERWRITE|MDB_NODUPDATA|MDB_RESERVE|MDB_APPEND|MDB_APPENDDUP)) != flags)
+	if (flags & ~(MDB_NOOVERWRITE|MDB_NODUPDATA|MDB_RESERVE|MDB_APPEND|MDB_APPENDDUP))
 		return EINVAL;
 
 	mdb_cursor_init(&mc, txn, dbi, &mx);
@@ -9151,7 +9148,7 @@ mdb_env_copy(MDB_env *env, const char *path)
 int ESECT
 mdb_env_set_flags(MDB_env *env, unsigned int flag, int onoff)
 {
-	if ((flag & CHANGEABLE) != flag)
+	if (flag & ~CHANGEABLE)
 		return EINVAL;
 	if (onoff)
 		env->me_flags |= flag;
@@ -9300,7 +9297,7 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db
 	unsigned int unused = 0, seq;
 	size_t len;
 
-	if ((flags & VALID_FLAGS) != flags)
+	if (flags & ~VALID_FLAGS)
 		return EINVAL;
 	if (txn->mt_flags & MDB_TXN_ERROR)
 		return MDB_BAD_TXN;

From ebb8b671095de788502807ad4074274a90eddecd Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 4 Jul 2015 13:49:12 +0200
Subject: [PATCH 095/504] Add DB_USRVALID, to avoid 'dbi == FREE_DBI' tests

---
 libraries/liblmdb/mdb.c | 38 ++++++++++++++++++++------------------
 1 file changed, 20 insertions(+), 18 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index c8f26bf21b..f949e1fd84 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1098,6 +1098,7 @@ struct MDB_txn {
 #define DB_STALE	0x02		/**< Named-DB record is older than txnID */
 #define DB_NEW		0x04		/**< Named-DB handle opened in this txn */
 #define DB_VALID	0x08		/**< DB handle is valid, see also #MDB_VALID */
+#define DB_USRVALID	0x10		/**< As #DB_VALID, but not set for #FREE_DBI */
 /** @} */
 	/** In write txns, array of cursors for each DB */
 	MDB_cursor	**mt_cursors;
@@ -1290,8 +1291,8 @@ typedef struct MDB_ntxn {
 #define MAX_WRITE		(0x80000000U >> (sizeof(ssize_t) == 4))
 
 	/** Check \b txn and \b dbi arguments to a function */
-#define TXN_DBI_EXIST(txn, dbi) \
-	((txn) && (dbi) < (txn)->mt_numdbs && ((txn)->mt_dbflags[dbi] & DB_VALID))
+#define TXN_DBI_EXIST(txn, dbi, validity) \
+	((txn) && (dbi)<(txn)->mt_numdbs && ((txn)->mt_dbflags[dbi] & (validity)))
 
 	/** Check for misused \b dbi handles */
 #define TXN_DBI_CHANGED(txn, dbi) \
@@ -2724,9 +2725,10 @@ mdb_txn_renew0(MDB_txn *txn)
 	for (i=2; i<txn->mt_numdbs; i++) {
 		x = env->me_dbflags[i];
 		txn->mt_dbs[i].md_flags = x & PERSISTENT_FLAGS;
-		txn->mt_dbflags[i] = (x & MDB_VALID) ? DB_VALID|DB_STALE : 0;
+		txn->mt_dbflags[i] = (x & MDB_VALID) ? DB_VALID|DB_USRVALID|DB_STALE : 0;
 	}
-	txn->mt_dbflags[0] = txn->mt_dbflags[1] = DB_VALID;
+	txn->mt_dbflags[MAIN_DBI] = DB_VALID|DB_USRVALID;
+	txn->mt_dbflags[FREE_DBI] = DB_VALID;
 
 	if (env->me_maxpg < txn->mt_next_pgno) {
 		mdb_txn_reset0(txn, "renew0-mapfail");
@@ -5573,7 +5575,7 @@ mdb_get(MDB_txn *txn, MDB_dbi dbi,
 
 	DPRINTF(("===> get db %u key [%s]", dbi, DKEY(key)));
 
-	if (!key || !data || dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi))
+	if (!key || !data || !TXN_DBI_EXIST(txn, dbi, DB_USRVALID))
 		return EINVAL;
 
 	if (txn->mt_flags & MDB_TXN_ERROR)
@@ -7286,7 +7288,7 @@ mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node)
 	}
 	DPRINTF(("Sub-db -%u root page %"Z"u", mx->mx_cursor.mc_dbi,
 		mx->mx_db.md_root));
-	mx->mx_dbflag = DB_VALID|DB_DIRTY; /* DB_DIRTY guides mdb_cursor_touch */
+	mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DIRTY; /* DB_DIRTY guides mdb_cursor_touch */
 #if UINT_MAX < SIZE_MAX
 	if (mx->mx_dbx.md_cmp == mdb_cmp_int && mx->mx_db.md_pad == sizeof(size_t))
 		mx->mx_dbx.md_cmp = mdb_cmp_clong;
@@ -7327,7 +7329,7 @@ mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **ret)
 	MDB_cursor	*mc;
 	size_t size = sizeof(MDB_cursor);
 
-	if (!ret || !TXN_DBI_EXIST(txn, dbi))
+	if (!ret || !TXN_DBI_EXIST(txn, dbi, DB_VALID))
 		return EINVAL;
 
 	if (txn->mt_flags & MDB_TXN_ERROR)
@@ -7359,7 +7361,7 @@ mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **ret)
 int
 mdb_cursor_renew(MDB_txn *txn, MDB_cursor *mc)
 {
-	if (!mc || !TXN_DBI_EXIST(txn, mc->mc_dbi))
+	if (!mc || !TXN_DBI_EXIST(txn, mc->mc_dbi, DB_VALID))
 		return EINVAL;
 
 	if ((mc->mc_flags & C_UNTRACK) || txn->mt_cursors)
@@ -8108,7 +8110,7 @@ int
 mdb_del(MDB_txn *txn, MDB_dbi dbi,
     MDB_val *key, MDB_val *data)
 {
-	if (!key || dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi))
+	if (!key || !TXN_DBI_EXIST(txn, dbi, DB_USRVALID))
 		return EINVAL;
 
 	if (txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_ERROR))
@@ -8575,7 +8577,7 @@ mdb_put(MDB_txn *txn, MDB_dbi dbi,
 	MDB_cursor mc;
 	MDB_xcursor mx;
 
-	if (!key || !data || dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi))
+	if (!key || !data || !TXN_DBI_EXIST(txn, dbi, DB_USRVALID))
 		return EINVAL;
 
 	if (flags & ~(MDB_NOOVERWRITE|MDB_NODUPDATA|MDB_RESERVE|MDB_APPEND|MDB_APPENDDUP))
@@ -9345,7 +9347,7 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db
 		return (flags & MDB_CREATE) ? MDB_INCOMPATIBLE : MDB_NOTFOUND;
 
 	/* Find the DB info */
-	dbflag = DB_NEW|DB_VALID;
+	dbflag = DB_NEW|DB_VALID|DB_USRVALID;
 	exact = 0;
 	key.mv_size = len;
 	key.mv_data = (void *)name;
@@ -9393,7 +9395,7 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db
 
 int mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *arg)
 {
-	if (!arg || !TXN_DBI_EXIST(txn, dbi))
+	if (!arg || !TXN_DBI_EXIST(txn, dbi, DB_VALID))
 		return EINVAL;
 
 	if (txn->mt_flags & MDB_TXN_ERROR)
@@ -9427,7 +9429,7 @@ void mdb_dbi_close(MDB_env *env, MDB_dbi dbi)
 int mdb_dbi_flags(MDB_txn *txn, MDB_dbi dbi, unsigned int *flags)
 {
 	/* We could return the flags for the FREE_DBI too but what's the point? */
-	if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi))
+	if (!TXN_DBI_EXIST(txn, dbi, DB_USRVALID))
 		return EINVAL;
 	*flags = txn->mt_dbs[dbi].md_flags & PERSISTENT_FLAGS;
 	return MDB_SUCCESS;
@@ -9527,7 +9529,7 @@ int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del)
 	MDB_cursor *mc, *m2;
 	int rc;
 
-	if ((unsigned)del > 1 || dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi))
+	if ((unsigned)del > 1 || !TXN_DBI_EXIST(txn, dbi, DB_USRVALID))
 		return EINVAL;
 
 	if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY))
@@ -9575,7 +9577,7 @@ leave:
 
 int mdb_set_compare(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp)
 {
-	if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi))
+	if (!TXN_DBI_EXIST(txn, dbi, DB_USRVALID))
 		return EINVAL;
 
 	txn->mt_dbxs[dbi].md_cmp = cmp;
@@ -9584,7 +9586,7 @@ int mdb_set_compare(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp)
 
 int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp)
 {
-	if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi))
+	if (!TXN_DBI_EXIST(txn, dbi, DB_USRVALID))
 		return EINVAL;
 
 	txn->mt_dbxs[dbi].md_dcmp = cmp;
@@ -9593,7 +9595,7 @@ int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp)
 
 int mdb_set_relfunc(MDB_txn *txn, MDB_dbi dbi, MDB_rel_func *rel)
 {
-	if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi))
+	if (!TXN_DBI_EXIST(txn, dbi, DB_USRVALID))
 		return EINVAL;
 
 	txn->mt_dbxs[dbi].md_rel = rel;
@@ -9602,7 +9604,7 @@ int mdb_set_relfunc(MDB_txn *txn, MDB_dbi dbi, MDB_rel_func *rel)
 
 int mdb_set_relctx(MDB_txn *txn, MDB_dbi dbi, void *ctx)
 {
-	if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi))
+	if (!TXN_DBI_EXIST(txn, dbi, DB_USRVALID))
 		return EINVAL;
 
 	txn->mt_dbxs[dbi].md_relctx = ctx;

From ad87d6a3f45d8ba2c00336bcdea1df5ebd17162a Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 4 Jul 2015 13:50:21 +0200
Subject: [PATCH 096/504] More ESECT declarations

---
 libraries/liblmdb/mdb.c | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index f949e1fd84..61141e9b4a 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1377,7 +1377,7 @@ static int mdb_sec_inited;
 #endif
 
 /** Return the library version info. */
-char *
+char * ESECT
 mdb_version(int *major, int *minor, int *patch)
 {
 	if (major) *major = MDB_VERSION_MAJOR;
@@ -1469,7 +1469,7 @@ mdb_strerror(int err)
 # define mdb_assert0(env, expr, expr_txt) ((expr) ? (void)0 : \
 		mdb_assert_fail(env, expr_txt, mdb_func_, __FILE__, __LINE__))
 
-static void
+static void ESECT
 mdb_assert_fail(MDB_env *env, const char *expr_txt,
 	const char *func, const char *file, int line)
 {
@@ -4412,7 +4412,7 @@ mdb_hash_val(MDB_val *val, mdb_hash_t hval)
  */
 static const char mdb_a85[]= "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~";
 
-static void
+static void ESECT
 mdb_pack85(unsigned long l, char *out)
 {
 	int i;
@@ -4423,7 +4423,7 @@ mdb_pack85(unsigned long l, char *out)
 	}
 }
 
-static void
+static void ESECT
 mdb_hash_enc(MDB_val *val, char *encbuf)
 {
 	mdb_hash_t h = mdb_hash_val(val, MDB_HASH_INIT);
@@ -9393,7 +9393,8 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db
 	return rc;
 }
 
-int mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *arg)
+int ESECT
+mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *arg)
 {
 	if (!arg || !TXN_DBI_EXIST(txn, dbi, DB_VALID))
 		return EINVAL;
@@ -9706,7 +9707,8 @@ mdb_reader_check(MDB_env *env, int *dead)
 }
 
 /** As #mdb_reader_check(). rlocked = <caller locked the reader mutex>. */
-static int mdb_reader_check0(MDB_env *env, int rlocked, int *dead)
+static int ESECT
+mdb_reader_check0(MDB_env *env, int rlocked, int *dead)
 {
 	mdb_mutexref_t rmutex = rlocked ? NULL : env->me_rmutex;
 	unsigned int i, j, rdrs;
@@ -9765,7 +9767,8 @@ static int mdb_reader_check0(MDB_env *env, int rlocked, int *dead)
  * @param[in] rc	LOCK_MUTEX0() error (nonzero)
  * @return 0 on success with the mutex locked, or an error code on failure.
  */
-static int mdb_mutex_failed(MDB_env *env, mdb_mutexref_t mutex, int rc)
+static int ESECT
+mdb_mutex_failed(MDB_env *env, mdb_mutexref_t mutex, int rc)
 {
 	int toggle, rlocked, rc2;
 

From 749633e48a80bfa6f5ca297769b6b9ff65ad3365 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 7 Jul 2015 13:41:32 +0100
Subject: [PATCH 097/504] ITS#8190 fix cursor EOF bug

---
 libraries/liblmdb/mdb.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 61141e9b4a..b576c82c34 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5924,8 +5924,10 @@ set2:
 
 	if (leaf == NULL) {
 		DPUTS("===> inexact leaf not found, goto sibling");
-		if ((rc = mdb_cursor_sibling(mc, 1)) != MDB_SUCCESS)
+		if ((rc = mdb_cursor_sibling(mc, 1)) != MDB_SUCCESS) {
+			mc->mc_flags |= C_EOF;
 			return rc;		/* no entries matched */
+		}
 		mp = mc->mc_pg[mc->mc_top];
 		mdb_cassert(mc, IS_LEAF(mp));
 		leaf = NODEPTR(mp, 0);

From 45e405528b0dc1dacdabcb6675bfa70432cd6b43 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 11 Jul 2015 21:01:40 +0200
Subject: [PATCH 098/504] ITS#8181 Verify that records are/aren't DBs.

Except we don't catch the user passing F_SUBDATA to
mdb_cursor_<put/del>, like an internal LMDB call.
---
 libraries/liblmdb/lmdb.h | 14 +++++++++++---
 libraries/liblmdb/mdb.c  | 17 ++++++++++++++---
 2 files changed, 25 insertions(+), 6 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index deb6812578..c27f78bfb8 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -421,7 +421,14 @@ typedef enum MDB_cursor_op {
 #define MDB_PAGE_FULL	(-30786)
 	/** Database contents grew beyond environment mapsize */
 #define MDB_MAP_RESIZED	(-30785)
-	/** MDB_INCOMPATIBLE: Operation and DB incompatible, or DB flags changed */
+	/** Operation and DB incompatible, or DB type changed. This can mean:
+	 *	<ul>
+	 *	<li>The operation expects an #MDB_DUPSORT / #MDB_DUPFIXED database.
+	 *	<li>Opening a named DB when the unnamed DB has #MDB_DUPSORT / #MDB_INTEGERKEY.
+	 *	<li>Accessing a data record as a database, or vice versa.
+	 *	<li>The database was dropped and recreated with different flags.
+	 *	</ul>
+	 */
 #define MDB_INCOMPATIBLE	(-30784)
 	/** Invalid reuse of reader locktable slot */
 #define MDB_BAD_RSLOT		(-30783)
@@ -1057,8 +1064,9 @@ int  mdb_txn_renew(MDB_txn *txn);
 	 * any other transaction in the process may use this function.
 	 *
 	 * To use named databases (with name != NULL), #mdb_env_set_maxdbs()
-	 * must be called before opening the environment.  Database names
-	 * are kept as keys in the unnamed database.
+	 * must be called before opening the environment.  Database names are
+	 * keys in the unnamed database, and may be read but not written.
+	 *
 	 * @param[in] txn A transaction handle returned by #mdb_txn_begin()
 	 * @param[in] name The name of the database to open. If only a single
 	 * 	database is needed in the environment, this value may be NULL.
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index b576c82c34..9ccac2ca8d 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -3519,7 +3519,8 @@ mdb_txn_commit(MDB_txn *txn)
 					goto fail;
 				}
 				data.mv_data = &txn->mt_dbs[i];
-				rc = mdb_cursor_put(&mc, &txn->mt_dbxs[i].md_name, &data, 0);
+				rc = mdb_cursor_put(&mc, &txn->mt_dbxs[i].md_name, &data,
+					F_SUBDATA);
 				if (rc)
 					goto fail;
 			}
@@ -5413,6 +5414,8 @@ mdb_page_search(MDB_cursor *mc, MDB_val *key, int flags)
 						&mc->mc_dbx->md_name, &exact);
 					if (!exact)
 						return MDB_NOTFOUND;
+					if ((leaf->mn_flags & (F_DUPDATA|F_SUBDATA)) != F_SUBDATA)
+						return MDB_INCOMPATIBLE; /* not a named DB */
 					rc = mdb_node_read(mc->mc_txn, leaf, &data);
 					if (rc)
 						return rc;
@@ -6604,6 +6607,9 @@ prep_subDB:
 			goto new_sub;
 		}
 current:
+		/* LMDB passes F_SUBDATA in 'flags' to write a DB record */
+		if ((leaf->mn_flags ^ flags) & F_SUBDATA)
+			return MDB_INCOMPATIBLE;
 		/* overflow page overwrites need special handling */
 		if (F_ISSET(leaf->mn_flags, F_BIGDATA)) {
 			MDB_page *omp;
@@ -6874,6 +6880,11 @@ mdb_cursor_del(MDB_cursor *mc, unsigned int flags)
 				goto fail;
 		}
 	}
+	/* LMDB passes F_SUBDATA in 'flags' to delete a DB record */
+	else if ((leaf->mn_flags ^ flags) & F_SUBDATA) {
+		rc = MDB_INCOMPATIBLE;
+		goto fail;
+	}
 
 	/* add overflow pages to free list */
 	if (F_ISSET(leaf->mn_flags, F_BIGDATA)) {
@@ -9358,7 +9369,7 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db
 	if (rc == MDB_SUCCESS) {
 		/* make sure this is actually a DB */
 		MDB_node *node = NODEPTR(mc.mc_pg[mc.mc_top], mc.mc_ki[mc.mc_top]);
-		if (!(node->mn_flags & F_SUBDATA))
+		if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) != F_SUBDATA)
 			return MDB_INCOMPATIBLE;
 	} else if (rc == MDB_NOTFOUND && (flags & MDB_CREATE)) {
 		/* Create if requested */
@@ -9554,7 +9565,7 @@ int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del)
 
 	/* Can't delete the main DB */
 	if (del && dbi > MAIN_DBI) {
-		rc = mdb_del0(txn, MAIN_DBI, &mc->mc_dbx->md_name, NULL, 0);
+		rc = mdb_del0(txn, MAIN_DBI, &mc->mc_dbx->md_name, NULL, F_SUBDATA);
 		if (!rc) {
 			txn->mt_dbflags[dbi] = DB_STALE;
 			mdb_dbi_close(txn->mt_env, dbi);

From 62aabaa4aa8d3abcc345a85ff004baaa663bab57 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 11 Jul 2015 21:09:36 +0200
Subject: [PATCH 099/504] Simpler mdb_node_add()

---
 libraries/liblmdb/mdb.c | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 9ccac2ca8d..a0b77663df 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7016,6 +7016,7 @@ mdb_node_add(MDB_cursor *mc, indx_t indx,
 	MDB_node	*node;
 	MDB_page	*mp = mc->mc_pg[mc->mc_top];
 	MDB_page	*ofp = NULL;		/* overflow page */
+	void		*ndata;
 	DKBUF;
 
 	mdb_cassert(mc, mp->mp_upper >= mp->mp_lower);
@@ -7046,7 +7047,7 @@ mdb_node_add(MDB_cursor *mc, indx_t indx,
 	if (key != NULL)
 		node_size += key->mv_size;
 	if (IS_LEAF(mp)) {
-		mdb_cassert(mc, data);
+		mdb_cassert(mc, key && data);
 		if (F_ISSET(flags, F_BIGDATA)) {
 			/* Data already on overflow page. */
 			node_size += sizeof(pgno_t);
@@ -7097,23 +7098,21 @@ update:
 		memcpy(NODEKEY(node), key->mv_data, key->mv_size);
 
 	if (IS_LEAF(mp)) {
-		mdb_cassert(mc, key);
+		ndata = NODEDATA(node);
 		if (ofp == NULL) {
 			if (F_ISSET(flags, F_BIGDATA))
-				memcpy(node->mn_data + key->mv_size, data->mv_data,
-				    sizeof(pgno_t));
+				memcpy(ndata, data->mv_data, sizeof(pgno_t));
 			else if (F_ISSET(flags, MDB_RESERVE))
-				data->mv_data = node->mn_data + key->mv_size;
+				data->mv_data = ndata;
 			else
-				memcpy(node->mn_data + key->mv_size, data->mv_data,
-				    data->mv_size);
+				memcpy(ndata, data->mv_data, data->mv_size);
 		} else {
-			memcpy(node->mn_data + key->mv_size, &ofp->mp_pgno,
-			    sizeof(pgno_t));
+			memcpy(ndata, &ofp->mp_pgno, sizeof(pgno_t));
+			ndata = METADATA(ofp);
 			if (F_ISSET(flags, MDB_RESERVE))
-				data->mv_data = METADATA(ofp);
+				data->mv_data = ndata;
 			else
-				memcpy(METADATA(ofp), data->mv_data, data->mv_size);
+				memcpy(ndata, data->mv_data, data->mv_size);
 		}
 	}
 

From 91fc43f17cec8ffa05a5c841831df1cfa0511469 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 11 Jul 2015 21:10:33 +0200
Subject: [PATCH 100/504] Factor me_metas[toggle] out to mdb_env_pick_meta()

---
 libraries/liblmdb/mdb.c | 53 ++++++++++++++++++++++-------------------
 1 file changed, 28 insertions(+), 25 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index a0b77663df..83ea5e474b 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1318,7 +1318,7 @@ static int	mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata,
 				pgno_t newpgno, unsigned int nflags);
 
 static int  mdb_env_read_header(MDB_env *env, MDB_meta *meta);
-static int  mdb_env_pick_meta(const MDB_env *env);
+static MDB_meta *mdb_env_pick_meta(const MDB_env *env);
 static int  mdb_env_write_meta(MDB_txn *txn);
 #ifdef MDB_USE_POSIX_MUTEX /* Drop unused excl arg */
 # define mdb_env_close0(env, excl) mdb_env_close1(env)
@@ -2622,7 +2622,7 @@ mdb_txn_renew0(MDB_txn *txn)
 
 	if ((flags &= MDB_TXN_RDONLY) != 0) {
 		if (!ti) {
-			meta = env->me_metas[ mdb_env_pick_meta(env) ];
+			meta = mdb_env_pick_meta(env);
 			txn->mt_txnid = meta->mm_txnid;
 			txn->mt_u.reader = NULL;
 		} else {
@@ -2691,7 +2691,7 @@ mdb_txn_renew0(MDB_txn *txn)
 			txn->mt_txnid = ti->mti_txnid;
 			meta = env->me_metas[txn->mt_txnid & 1];
 		} else {
-			meta = env->me_metas[ mdb_env_pick_meta(env) ];
+			meta = mdb_env_pick_meta(env);
 			txn->mt_txnid = meta->mm_txnid;
 		}
 		txn->mt_txnid++;
@@ -3831,12 +3831,13 @@ done:
 
 /** Check both meta pages to see which one is newer.
  * @param[in] env the environment handle
- * @return meta toggle (0 or 1).
+ * @return newest #MDB_meta.
  */
-static int
+static MDB_meta *
 mdb_env_pick_meta(const MDB_env *env)
 {
-	return (env->me_metas[0]->mm_txnid < env->me_metas[1]->mm_txnid);
+	MDB_meta *const *metas = env->me_metas;
+	return metas[ metas[0]->mm_txnid < metas[1]->mm_txnid ];
 }
 
 int ESECT
@@ -3963,7 +3964,7 @@ mdb_env_set_mapsize(MDB_env *env, size_t size)
 		void *old;
 		if (env->me_txn)
 			return EINVAL;
-		meta = env->me_metas[mdb_env_pick_meta(env)];
+		meta = mdb_env_pick_meta(env);
 		if (!size)
 			size = meta->mm_mapsize;
 		{
@@ -4170,12 +4171,12 @@ mdb_env_open2(MDB_env *env)
 
 #if MDB_DEBUG
 	{
-		int toggle = mdb_env_pick_meta(env);
-		MDB_db *db = &env->me_metas[toggle]->mm_dbs[MAIN_DBI];
+		MDB_meta *meta = mdb_env_pick_meta(env);
+		MDB_db *db = &meta->mm_dbs[MAIN_DBI];
 
 		DPRINTF(("opened database version %u, pagesize %u",
-			env->me_metas[0]->mm_version, env->me_psize));
-		DPRINTF(("using meta page %d",    toggle));
+			meta->mm_version, env->me_psize));
+		DPRINTF(("using meta page %d",    (int) (meta->mm_txnid & 1)));
 		DPRINTF(("depth: %u",             db->md_depth));
 		DPRINTF(("entries: %"Z"u",        db->md_entries));
 		DPRINTF(("branch pages: %"Z"u",   db->md_branch_pages));
@@ -4262,9 +4263,10 @@ PIMAGE_TLS_CALLBACK mdb_tls_cbp = mdb_tls_callback;
 static int ESECT
 mdb_env_share_locks(MDB_env *env, int *excl)
 {
-	int rc = 0, toggle = mdb_env_pick_meta(env);
+	int rc = 0;
+	MDB_meta *meta = mdb_env_pick_meta(env);
 
-	env->me_txns->mti_txnid = env->me_metas[toggle]->mm_txnid;
+	env->me_txns->mti_txnid = meta->mm_txnid;
 
 #ifdef _WIN32
 	{
@@ -9249,32 +9251,32 @@ mdb_stat0(MDB_env *env, MDB_db *db, MDB_stat *arg)
 int ESECT
 mdb_env_stat(MDB_env *env, MDB_stat *arg)
 {
-	int toggle;
+	MDB_meta *meta;
 
 	if (env == NULL || arg == NULL)
 		return EINVAL;
 
-	toggle = mdb_env_pick_meta(env);
+	meta = mdb_env_pick_meta(env);
 
-	return mdb_stat0(env, &env->me_metas[toggle]->mm_dbs[MAIN_DBI], arg);
+	return mdb_stat0(env, &meta->mm_dbs[MAIN_DBI], arg);
 }
 
 int ESECT
 mdb_env_info(MDB_env *env, MDB_envinfo *arg)
 {
-	int toggle;
+	MDB_meta *meta;
 
 	if (env == NULL || arg == NULL)
 		return EINVAL;
 
-	toggle = mdb_env_pick_meta(env);
-	arg->me_mapaddr = env->me_metas[toggle]->mm_address;
+	meta = mdb_env_pick_meta(env);
+	arg->me_mapaddr = meta->mm_address;
+	arg->me_last_pgno = meta->mm_last_pg;
+	arg->me_last_txnid = meta->mm_txnid;
+
 	arg->me_mapsize = env->me_mapsize;
 	arg->me_maxreaders = env->me_maxreaders;
 	arg->me_numreaders = env->me_txns ? env->me_txns->mti_numreaders : 0;
-
-	arg->me_last_pgno = env->me_metas[toggle]->mm_last_pg;
-	arg->me_last_txnid = env->me_metas[toggle]->mm_txnid;
 	return MDB_SUCCESS;
 }
 
@@ -9782,7 +9784,8 @@ mdb_reader_check0(MDB_env *env, int rlocked, int *dead)
 static int ESECT
 mdb_mutex_failed(MDB_env *env, mdb_mutexref_t mutex, int rc)
 {
-	int toggle, rlocked, rc2;
+	int rlocked, rc2;
+	MDB_meta *meta;
 
 	if (rc == MDB_OWNERDEAD) {
 		/* We own the mutex. Clean up after dead previous owner. */
@@ -9792,8 +9795,8 @@ mdb_mutex_failed(MDB_env *env, mdb_mutexref_t mutex, int rc)
 			/* Keep mti_txnid updated, otherwise next writer can
 			 * overwrite data which latest meta page refers to.
 			 */
-			toggle = mdb_env_pick_meta(env);
-			env->me_txns->mti_txnid = env->me_metas[toggle]->mm_txnid;
+			meta = mdb_env_pick_meta(env);
+			env->me_txns->mti_txnid = meta->mm_txnid;
 			/* env is hosed if the dead thread was ours */
 			if (env->me_txn) {
 				env->me_flags |= MDB_FATAL_ERROR;

From 90f6fc4ba7d60f602acd1ef63e4c7d939fb514e1 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sun, 19 Jul 2015 21:18:31 +0200
Subject: [PATCH 101/504] ITS#8200 Fix mdb_midl_shrink() usage, return void

---
 libraries/liblmdb/mdb.c  | 8 ++++----
 libraries/liblmdb/midl.c | 4 +---
 libraries/liblmdb/midl.h | 3 +--
 3 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 83ea5e474b..a2a68a1641 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2947,8 +2947,8 @@ mdb_txn_reset0(MDB_txn *txn, const char *act)
 		}
 
 		if (!txn->mt_parent) {
-			if (mdb_midl_shrink(&txn->mt_free_pgs))
-				env->me_free_pgs = txn->mt_free_pgs;
+			mdb_midl_shrink(&txn->mt_free_pgs);
+			env->me_free_pgs = txn->mt_free_pgs;
 			/* me_pgstate: */
 			env->me_pghead = NULL;
 			env->me_pglast = 0;
@@ -3533,8 +3533,8 @@ mdb_txn_commit(MDB_txn *txn)
 
 	mdb_midl_free(env->me_pghead);
 	env->me_pghead = NULL;
-	if (mdb_midl_shrink(&txn->mt_free_pgs))
-		env->me_free_pgs = txn->mt_free_pgs;
+	mdb_midl_shrink(&txn->mt_free_pgs);
+	env->me_free_pgs = txn->mt_free_pgs;
 
 #if (MDB_DEBUG) > 2
 	mdb_audit(txn);
diff --git a/libraries/liblmdb/midl.c b/libraries/liblmdb/midl.c
index 16782dcafc..57a9d4920e 100644
--- a/libraries/liblmdb/midl.c
+++ b/libraries/liblmdb/midl.c
@@ -116,7 +116,7 @@ void mdb_midl_free(MDB_IDL ids)
 		free(ids-1);
 }
 
-int mdb_midl_shrink( MDB_IDL *idp )
+void mdb_midl_shrink( MDB_IDL *idp )
 {
 	MDB_IDL ids = *idp;
 	if (*(--ids) > MDB_IDL_UM_MAX &&
@@ -124,9 +124,7 @@ int mdb_midl_shrink( MDB_IDL *idp )
 	{
 		*ids++ = MDB_IDL_UM_MAX;
 		*idp = ids;
-		return 1;
 	}
-	return 0;
 }
 
 static int mdb_midl_grow( MDB_IDL *idp, int num )
diff --git a/libraries/liblmdb/midl.h b/libraries/liblmdb/midl.h
index 9b041d72d1..2331e78398 100644
--- a/libraries/liblmdb/midl.h
+++ b/libraries/liblmdb/midl.h
@@ -98,9 +98,8 @@ void mdb_midl_free(MDB_IDL ids);
 	/** Shrink an IDL.
 	 * Return the IDL to the default size if it has grown larger.
 	 * @param[in,out] idp	Address of the IDL to shrink.
-	 * @return	0 on no change, non-zero if shrunk.
 	 */
-int mdb_midl_shrink(MDB_IDL *idp);
+void mdb_midl_shrink(MDB_IDL *idp);
 
 	/** Make room for num additional elements in an IDL.
 	 * @param[in,out] idp	Address of the IDL.

From b6973a3324144c8c5b0dfc3da482aa1cf9ac8ce2 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sun, 19 Jul 2015 21:30:12 +0200
Subject: [PATCH 102/504] Move code into mdb_txn_end(). Was mdb_txn_reset0.

Side effects:
* Clean txn up a bit even before freeing it.
* Tweak DEBUG output at txn end. Add DEBUG after commit(writer).
---
 libraries/liblmdb/mdb.c | 97 ++++++++++++++++++++---------------------
 1 file changed, 48 insertions(+), 49 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index a2a68a1641..32508dcd03 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1302,6 +1302,19 @@ static int  mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp);
 static int  mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp);
 static int  mdb_page_touch(MDB_cursor *mc);
 
+#define MDB_END_NAMES {"committed", "empty-commit", "abort", "reset", \
+	"reset-tmp", "fail-begin", "fail-beginchild"}
+enum {
+	/* mdb_txn_end operation number, for logging */
+	MDB_END_COMMITTED, MDB_END_EMPTY_COMMIT, MDB_END_ABORT, MDB_END_RESET,
+	MDB_END_RESET_TMP, MDB_END_FAIL_BEGIN, MDB_END_FAIL_BEGINCHILD
+};
+#define MDB_END_OPMASK	0x0F	/**< mask for #mdb_txn_end() operation number */
+#define MDB_END_UPDATE	0x10	/**< update env state (DBIs) */
+#define MDB_END_FREE	0x20	/**< free txn unless it is #MDB_env.%me_txn0 */
+#define MDB_END_SLOT MDB_NOTLS	/**< release any reader slot if #MDB_NOTLS */
+static void mdb_txn_end(MDB_txn *txn, unsigned mode);
+
 static int  mdb_page_get(MDB_txn *txn, pgno_t pgno, MDB_page **mp, int *lvl);
 static int  mdb_page_search_root(MDB_cursor *mc,
 			    MDB_val *key, int modify);
@@ -2546,12 +2559,6 @@ mdb_cursors_close(MDB_txn *txn, unsigned merge)
 	}
 }
 
-#if !(MDB_DEBUG)
-#define mdb_txn_reset0(txn, act) mdb_txn_reset0(txn)
-#endif
-static void
-mdb_txn_reset0(MDB_txn *txn, const char *act);
-
 #if !(MDB_PIDLOCK)		/* Currently the same as defined(_WIN32) */
 enum Pidlock_op {
 	Pidset, Pidcheck
@@ -2731,11 +2738,7 @@ mdb_txn_renew0(MDB_txn *txn)
 	txn->mt_dbflags[FREE_DBI] = DB_VALID;
 
 	if (env->me_maxpg < txn->mt_next_pgno) {
-		mdb_txn_reset0(txn, "renew0-mapfail");
-		if (new_notls) {
-			txn->mt_u.reader->mr_pid = 0;
-			txn->mt_u.reader = NULL;
-		}
+		mdb_txn_end(txn, new_notls /*0 or MDB_END_SLOT*/ | MDB_END_FAIL_BEGIN);
 		return MDB_MAP_RESIZED;
 	}
 
@@ -2850,7 +2853,7 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 		if (!rc)
 			rc = mdb_cursor_shadow(parent, txn);
 		if (rc)
-			mdb_txn_reset0(txn, "beginchild-fail");
+			mdb_txn_end(txn, MDB_END_FAIL_BEGINCHILD);
 	} else { /* MDB_RDONLY */
 		txn->mt_dbiseqs = env->me_dbiseqs;
 renew:
@@ -2913,35 +2916,44 @@ mdb_dbis_update(MDB_txn *txn, int keep)
 		env->me_numdbs = n;
 }
 
-/** Common code for #mdb_txn_reset() and #mdb_txn_abort().
+/** End a transaction, except successful commit of a nested transaction.
  * May be called twice for readonly txns: First reset it, then abort.
- * @param[in] txn the transaction handle to reset
- * @param[in] act why the transaction is being reset
+ * @param[in] txn the transaction handle to end
+ * @param[in] mode why and how to end the transaction
  */
 static void
-mdb_txn_reset0(MDB_txn *txn, const char *act)
+mdb_txn_end(MDB_txn *txn, unsigned mode)
 {
 	MDB_env	*env = txn->mt_env;
+#if MDB_DEBUG
+	static const char *const names[] = MDB_END_NAMES;
+#endif
 
-	/* Close any DBI handles opened in this txn */
-	mdb_dbis_update(txn, 0);
+	/* Export or close DBI handles opened in this txn */
+	mdb_dbis_update(txn, mode & MDB_END_UPDATE);
 
 	DPRINTF(("%s txn %"Z"u%c %p on mdbenv %p, root page %"Z"u",
-		act, txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w',
+		names[mode & MDB_END_OPMASK],
+		txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w',
 		(void *) txn, (void *)env, txn->mt_dbs[MAIN_DBI].md_root));
 
 	if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) {
 		if (txn->mt_u.reader) {
 			txn->mt_u.reader->mr_txnid = (txnid_t)-1;
-			if (!(env->me_flags & MDB_NOTLS))
+			if (!(env->me_flags & MDB_NOTLS)) {
 				txn->mt_u.reader = NULL; /* txn does not own reader */
+			} else if (mode & MDB_END_SLOT) {
+				txn->mt_u.reader->mr_pid = 0;
+				txn->mt_u.reader = NULL;
+			} /* else txn owns the slot until it does MDB_END_SLOT */
 		}
 		txn->mt_numdbs = 0;		/* close nothing if called again */
 		txn->mt_dbxs = NULL;	/* mark txn as reset */
 	} else {
 		pgno_t *pghead = env->me_pghead;
 
-		mdb_cursors_close(txn, 0);
+		if (!(mode & MDB_END_UPDATE)) /* !(already closed cursors) */
+			mdb_cursors_close(txn, 0);
 		if (!(env->me_flags & MDB_WRITEMAP)) {
 			mdb_dlist_free(txn);
 		}
@@ -2954,6 +2966,8 @@ mdb_txn_reset0(MDB_txn *txn, const char *act)
 			env->me_pglast = 0;
 
 			env->me_txn = NULL;
+			mode = 0;	/* txn == env->me_txn0, do not free() it */
+
 			/* The writer mutex was locked in mdb_txn_begin. */
 			if (env->me_txns)
 				UNLOCK_MUTEX(env->me_wmutex);
@@ -2967,6 +2981,9 @@ mdb_txn_reset0(MDB_txn *txn, const char *act)
 
 		mdb_midl_free(pghead);
 	}
+
+	if (mode & MDB_END_FREE)
+		free(txn);
 }
 
 void
@@ -2979,7 +2996,7 @@ mdb_txn_reset(MDB_txn *txn)
 	if (!(txn->mt_flags & MDB_TXN_RDONLY))
 		return;
 
-	mdb_txn_reset0(txn, "reset");
+	mdb_txn_end(txn, MDB_END_RESET);
 }
 
 void
@@ -2991,13 +3008,7 @@ mdb_txn_abort(MDB_txn *txn)
 	if (txn->mt_child)
 		mdb_txn_abort(txn->mt_child);
 
-	mdb_txn_reset0(txn, "abort");
-	/* Free reader slot tied to this txn (if MDB_NOTLS && writable FS) */
-	if ((txn->mt_flags & MDB_TXN_RDONLY) && txn->mt_u.reader)
-		txn->mt_u.reader->mr_pid = 0;
-
-	if (txn != txn->mt_env->me_txn0)
-		free(txn);
+	mdb_txn_end(txn, MDB_END_ABORT|MDB_END_SLOT|MDB_END_FREE);
 }
 
 /** Save the freelist as of this transaction to the freeDB.
@@ -3348,12 +3359,15 @@ int
 mdb_txn_commit(MDB_txn *txn)
 {
 	int		rc;
-	unsigned int i;
+	unsigned int i, end_mode;
 	MDB_env	*env;
 
 	if (txn == NULL)
 		return EINVAL;
 
+	/* mdb_txn_end() mode for a commit which writes nothing */
+	end_mode = MDB_END_EMPTY_COMMIT|MDB_END_UPDATE|MDB_END_SLOT|MDB_END_FREE;
+
 	if (txn->mt_child) {
 		rc = mdb_txn_commit(txn->mt_child);
 		if (rc)
@@ -3363,10 +3377,7 @@ mdb_txn_commit(MDB_txn *txn)
 	env = txn->mt_env;
 
 	if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) {
-		mdb_dbis_update(txn, 1);
-		txn->mt_numdbs = 2; /* so txn_abort() doesn't close any new handles */
-		mdb_txn_abort(txn);
-		return MDB_SUCCESS;
+		goto done;
 	}
 
 	if (F_ISSET(txn->mt_flags, MDB_TXN_ERROR)) {
@@ -3534,7 +3545,6 @@ mdb_txn_commit(MDB_txn *txn)
 	mdb_midl_free(env->me_pghead);
 	env->me_pghead = NULL;
 	mdb_midl_shrink(&txn->mt_free_pgs);
-	env->me_free_pgs = txn->mt_free_pgs;
 
 #if (MDB_DEBUG) > 2
 	mdb_audit(txn);
@@ -3547,21 +3557,10 @@ mdb_txn_commit(MDB_txn *txn)
 		goto fail;
 	if ((rc = mdb_env_write_meta(txn)))
 		goto fail;
-
-	/* Free P_LOOSE pages left behind in dirty_list */
-	if (!(env->me_flags & MDB_WRITEMAP))
-		mdb_dlist_free(txn);
+	end_mode = MDB_END_COMMITTED|MDB_END_UPDATE;
 
 done:
-	env->me_pglast = 0;
-	env->me_txn = NULL;
-	mdb_dbis_update(txn, 1);
-
-	if (env->me_txns)
-		UNLOCK_MUTEX(env->me_wmutex);
-	if (txn != env->me_txn0)
-		free(txn);
-
+	mdb_txn_end(txn, end_mode);
 	return MDB_SUCCESS;
 
 fail:
@@ -9007,7 +9006,7 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd)
 
 	if (env->me_txns) {
 		/* We must start the actual read txn after blocking writers */
-		mdb_txn_reset0(txn, "reset-stage1");
+		mdb_txn_end(txn, MDB_END_RESET_TMP);
 
 		/* Temporarily block writers until we snapshot the meta pages */
 		wmutex = env->me_wmutex;

From 3e301ec40025f17b52a7bcd66620c95ae3efef0f Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sun, 19 Jul 2015 21:31:25 +0200
Subject: [PATCH 103/504] Catch most uses of finished/parent txns.

* Add MDB_TXN_FINISHED, MDB_TXN_HAS_CHILD, MDB_TXN_BLOCKED.
* Clear mt_numdbs in writers, for TXN_DBI_EXIST() to catch.
  We already do in readers.
---
 libraries/liblmdb/lmdb.h |  2 +-
 libraries/liblmdb/mdb.c  | 57 ++++++++++++++++++++++++----------------
 2 files changed, 35 insertions(+), 24 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index c27f78bfb8..b72418e4fb 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -432,7 +432,7 @@ typedef enum MDB_cursor_op {
 #define MDB_INCOMPATIBLE	(-30784)
 	/** Invalid reuse of reader locktable slot */
 #define MDB_BAD_RSLOT		(-30783)
-	/** Transaction cannot recover - it must be aborted */
+	/** Transaction must abort, has a child, or is invalid */
 #define MDB_BAD_TXN			(-30782)
 	/** Unsupported size of key/DB name/data, or wrong DUPFIXED size */
 #define MDB_BAD_VALSIZE		(-30781)
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 32508dcd03..ec34cfec50 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1056,7 +1056,8 @@ typedef struct MDB_dbx {
 	 */
 struct MDB_txn {
 	MDB_txn		*mt_parent;		/**< parent of a nested txn */
-	MDB_txn		*mt_child;		/**< nested txn under this txn */
+	/** Nested txn under this txn, set together with flag #MDB_TXN_HAS_CHILD */
+	MDB_txn		*mt_child;
 	pgno_t		mt_next_pgno;	/**< next unallocated page */
 	/** The ID of this transaction. IDs are integers incrementing from 1.
 	 *	Only committed write transactions increment the ID. If a transaction
@@ -1104,8 +1105,9 @@ struct MDB_txn {
 	MDB_cursor	**mt_cursors;
 	/** Array of flags for each DB */
 	unsigned char	*mt_dbflags;
-	/**	Number of DB records in use. This number only ever increments;
-	 *	we don't decrement it when individual DB handles are closed.
+	/**	Number of DB records in use, or 0 when the txn is finished.
+	 *	This number only ever increments until the txn finishes; we
+	 *	don't decrement it when individual DB handles are closed.
 	 */
 	MDB_dbi		mt_numdbs;
 
@@ -1120,9 +1122,13 @@ struct MDB_txn {
 #define MDB_TXN_RDONLY		MDB_RDONLY	/**< read-only transaction */
 	/* internal txn flags */
 #define MDB_TXN_WRITEMAP	MDB_WRITEMAP	/**< copy of #MDB_env flag in writers */
+#define MDB_TXN_FINISHED	0x01		/**< txn is finished or never began */
 #define MDB_TXN_ERROR		0x02		/**< txn is unusable after an error */
 #define MDB_TXN_DIRTY		0x04		/**< must write, even if dirty list is empty */
 #define MDB_TXN_SPILLS		0x08		/**< txn or a parent has spilled pages */
+#define MDB_TXN_HAS_CHILD	0x10		/**< txn has an #MDB_txn.%mt_child */
+	/** most operations on the txn are currently illegal */
+#define MDB_TXN_BLOCKED		(MDB_TXN_FINISHED|MDB_TXN_ERROR|MDB_TXN_HAS_CHILD)
 /** @} */
 	unsigned int	mt_flags;		/**< @ref mdb_txn */
 	/** #dirty_list room: Array size - \#dirty pages visible to this txn.
@@ -1418,7 +1424,7 @@ static char *const mdb_errstr[] = {
 	"MDB_MAP_RESIZED: Database contents grew beyond environment mapsize",
 	"MDB_INCOMPATIBLE: Operation and DB incompatible, or DB flags changed",
 	"MDB_BAD_RSLOT: Invalid reuse of reader locktable slot",
-	"MDB_BAD_TXN: Transaction cannot recover - it must be aborted",
+	"MDB_BAD_TXN: Transaction must abort, has a child, or is invalid",
 	"MDB_BAD_VALSIZE: Unsupported size of key/DB name/data, or wrong DUPFIXED size",
 	"MDB_BAD_DBI: The specified DBI handle was closed/changed unexpectedly",
 };
@@ -2787,9 +2793,7 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 	if (parent) {
 		/* Nested transactions: Max 1 child, write txns only, no writemap */
 		flags |= parent->mt_flags;
-		if (parent->mt_child ||
-			(flags & (MDB_RDONLY|MDB_WRITEMAP|MDB_TXN_ERROR)))
-		{
+		if (flags & (MDB_RDONLY|MDB_WRITEMAP|MDB_TXN_BLOCKED)) {
 			return (parent->mt_flags & MDB_TXN_RDONLY) ? EINVAL : MDB_BAD_TXN;
 		}
 		/* Child txns save MDB_pgstate and use own copy of cursors */
@@ -2831,6 +2835,7 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 		txn->mt_u.dirty_list[0].mid = 0;
 		txn->mt_spill_pgs = NULL;
 		txn->mt_next_pgno = parent->mt_next_pgno;
+		parent->mt_flags |= MDB_TXN_HAS_CHILD;
 		parent->mt_child = txn;
 		txn->mt_parent = parent;
 		txn->mt_numdbs = parent->mt_numdbs;
@@ -2947,9 +2952,10 @@ mdb_txn_end(MDB_txn *txn, unsigned mode)
 				txn->mt_u.reader = NULL;
 			} /* else txn owns the slot until it does MDB_END_SLOT */
 		}
-		txn->mt_numdbs = 0;		/* close nothing if called again */
+		txn->mt_numdbs = 0;		/* prevent further DBI activity */
+		txn->mt_flags |= MDB_TXN_FINISHED;
 		txn->mt_dbxs = NULL;	/* mark txn as reset */
-	} else {
+	} else if (!F_ISSET(txn->mt_flags, MDB_TXN_FINISHED)) {
 		pgno_t *pghead = env->me_pghead;
 
 		if (!(mode & MDB_END_UPDATE)) /* !(already closed cursors) */
@@ -2958,6 +2964,9 @@ mdb_txn_end(MDB_txn *txn, unsigned mode)
 			mdb_dlist_free(txn);
 		}
 
+		txn->mt_numdbs = 0;
+		txn->mt_flags = MDB_TXN_FINISHED;
+
 		if (!txn->mt_parent) {
 			mdb_midl_shrink(&txn->mt_free_pgs);
 			env->me_free_pgs = txn->mt_free_pgs;
@@ -2973,6 +2982,7 @@ mdb_txn_end(MDB_txn *txn, unsigned mode)
 				UNLOCK_MUTEX(env->me_wmutex);
 		} else {
 			txn->mt_parent->mt_child = NULL;
+			txn->mt_parent->mt_flags &= ~MDB_TXN_HAS_CHILD;
 			env->me_pgstate = ((MDB_ntxn *)txn)->mnt_pgstate;
 			mdb_midl_free(txn->mt_free_pgs);
 			mdb_midl_free(txn->mt_spill_pgs);
@@ -3380,8 +3390,8 @@ mdb_txn_commit(MDB_txn *txn)
 		goto done;
 	}
 
-	if (F_ISSET(txn->mt_flags, MDB_TXN_ERROR)) {
-		DPUTS("error flag is set, can't commit");
+	if (txn->mt_flags & (MDB_TXN_FINISHED|MDB_TXN_ERROR)) {
+		DPUTS("txn has failed/finished, can't commit");
 		if (txn->mt_parent)
 			txn->mt_parent->mt_flags |= MDB_TXN_ERROR;
 		rc = MDB_BAD_TXN;
@@ -4842,6 +4852,7 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 				txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs);
 				txn->mt_env = env;
 				txn->mt_dbxs = env->me_dbxs;
+				txn->mt_flags = MDB_TXN_FINISHED;
 				env->me_txn0 = txn;
 			} else {
 				rc = ENOMEM;
@@ -5394,8 +5405,8 @@ mdb_page_search(MDB_cursor *mc, MDB_val *key, int flags)
 	/* Make sure the txn is still viable, then find the root from
 	 * the txn's db table and set it as the root of the cursor's stack.
 	 */
-	if (F_ISSET(mc->mc_txn->mt_flags, MDB_TXN_ERROR)) {
-		DPUTS("transaction has failed, must abort");
+	if (mc->mc_txn->mt_flags & MDB_TXN_BLOCKED) {
+		DPUTS("transaction may not be used now");
 		return MDB_BAD_TXN;
 	} else {
 		/* Make sure we're using an up-to-date root */
@@ -5582,7 +5593,7 @@ mdb_get(MDB_txn *txn, MDB_dbi dbi,
 	if (!key || !data || !TXN_DBI_EXIST(txn, dbi, DB_USRVALID))
 		return EINVAL;
 
-	if (txn->mt_flags & MDB_TXN_ERROR)
+	if (txn->mt_flags & MDB_TXN_BLOCKED)
 		return MDB_BAD_TXN;
 
 	mdb_cursor_init(&mc, txn, dbi, &mx);
@@ -6103,7 +6114,7 @@ mdb_cursor_get(MDB_cursor *mc, MDB_val *key, MDB_val *data,
 	if (mc == NULL)
 		return EINVAL;
 
-	if (mc->mc_txn->mt_flags & MDB_TXN_ERROR)
+	if (mc->mc_txn->mt_flags & MDB_TXN_BLOCKED)
 		return MDB_BAD_TXN;
 
 	switch (op) {
@@ -6333,7 +6344,7 @@ mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data,
 	nospill = flags & MDB_NOSPILL;
 	flags &= ~MDB_NOSPILL;
 
-	if (mc->mc_txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_ERROR))
+	if (mc->mc_txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_BLOCKED))
 		return (mc->mc_txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN;
 
 	if (key->mv_size-1 >= ENV_MAXKEY(env))
@@ -6815,7 +6826,7 @@ mdb_cursor_del(MDB_cursor *mc, unsigned int flags)
 	MDB_page	*mp;
 	int rc;
 
-	if (mc->mc_txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_ERROR))
+	if (mc->mc_txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_BLOCKED))
 		return (mc->mc_txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN;
 
 	if (!(mc->mc_flags & C_INITIALIZED))
@@ -7345,7 +7356,7 @@ mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **ret)
 	if (!ret || !TXN_DBI_EXIST(txn, dbi, DB_VALID))
 		return EINVAL;
 
-	if (txn->mt_flags & MDB_TXN_ERROR)
+	if (txn->mt_flags & MDB_TXN_BLOCKED)
 		return MDB_BAD_TXN;
 
 	/* Allow read access to the freelist */
@@ -7380,7 +7391,7 @@ mdb_cursor_renew(MDB_txn *txn, MDB_cursor *mc)
 	if ((mc->mc_flags & C_UNTRACK) || txn->mt_cursors)
 		return EINVAL;
 
-	if (txn->mt_flags & MDB_TXN_ERROR)
+	if (txn->mt_flags & MDB_TXN_BLOCKED)
 		return MDB_BAD_TXN;
 
 	mdb_cursor_init(mc, txn, mc->mc_dbi, mc->mc_xcursor);
@@ -7399,7 +7410,7 @@ mdb_cursor_count(MDB_cursor *mc, size_t *countp)
 	if (mc->mc_xcursor == NULL)
 		return MDB_INCOMPATIBLE;
 
-	if (mc->mc_txn->mt_flags & MDB_TXN_ERROR)
+	if (mc->mc_txn->mt_flags & MDB_TXN_BLOCKED)
 		return MDB_BAD_TXN;
 
 	if (!(mc->mc_flags & C_INITIALIZED))
@@ -8126,7 +8137,7 @@ mdb_del(MDB_txn *txn, MDB_dbi dbi,
 	if (!key || !TXN_DBI_EXIST(txn, dbi, DB_USRVALID))
 		return EINVAL;
 
-	if (txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_ERROR))
+	if (txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_BLOCKED))
 		return (txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN;
 
 	if (!F_ISSET(txn->mt_dbs[dbi].md_flags, MDB_DUPSORT)) {
@@ -9314,7 +9325,7 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db
 
 	if (flags & ~VALID_FLAGS)
 		return EINVAL;
-	if (txn->mt_flags & MDB_TXN_ERROR)
+	if (txn->mt_flags & MDB_TXN_BLOCKED)
 		return MDB_BAD_TXN;
 
 	/* main DB? */
@@ -9412,7 +9423,7 @@ mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *arg)
 	if (!arg || !TXN_DBI_EXIST(txn, dbi, DB_VALID))
 		return EINVAL;
 
-	if (txn->mt_flags & MDB_TXN_ERROR)
+	if (txn->mt_flags & MDB_TXN_BLOCKED)
 		return MDB_BAD_TXN;
 
 	if (txn->mt_dbflags[dbi] & DB_STALE) {

From 060b63a58dab60e0e460bb71ae4cc9c9571585ff Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sun, 19 Jul 2015 21:31:54 +0200
Subject: [PATCH 104/504] Cleanup: Drop !mt_dbxs hack, use MDB_TXN_FINISHED

---
 libraries/liblmdb/mdb.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index ec34cfec50..b67bf09bec 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2695,7 +2695,7 @@ mdb_txn_renew0(MDB_txn *txn)
 			txn->mt_u.reader = r;
 			meta = env->me_metas[txn->mt_txnid & 1];
 		}
-		txn->mt_dbxs = env->me_dbxs;	/* mostly static anyway */
+
 	} else {
 		/* Not yet touching txn == env->me_txn0, it may be active */
 		if (ti) {
@@ -2756,7 +2756,7 @@ mdb_txn_renew(MDB_txn *txn)
 {
 	int rc;
 
-	if (!txn || txn->mt_dbxs)	/* A reset txn has mt_dbxs==NULL */
+	if (!txn || !F_ISSET(txn->mt_flags, MDB_TXN_RDONLY|MDB_TXN_FINISHED))
 		return EINVAL;
 
 	if (txn->mt_env->me_flags & MDB_FATAL_ERROR) {
@@ -2813,6 +2813,7 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 		DPRINTF(("calloc: %s", strerror(errno)));
 		return ENOMEM;
 	}
+	txn->mt_dbxs = env->me_dbxs;	/* static */
 	txn->mt_dbs = (MDB_db *) ((char *)txn + tsize);
 	txn->mt_dbflags = (unsigned char *)txn + size - env->me_maxdbs;
 	txn->mt_flags = flags;
@@ -2839,7 +2840,6 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 		parent->mt_child = txn;
 		txn->mt_parent = parent;
 		txn->mt_numdbs = parent->mt_numdbs;
-		txn->mt_dbxs = parent->mt_dbxs;
 		memcpy(txn->mt_dbs, parent->mt_dbs, txn->mt_numdbs * sizeof(MDB_db));
 		/* Copy parent's mt_dbflags, but clear DB_NEW */
 		for (i=0; i<txn->mt_numdbs; i++)
@@ -2954,7 +2954,7 @@ mdb_txn_end(MDB_txn *txn, unsigned mode)
 		}
 		txn->mt_numdbs = 0;		/* prevent further DBI activity */
 		txn->mt_flags |= MDB_TXN_FINISHED;
-		txn->mt_dbxs = NULL;	/* mark txn as reset */
+
 	} else if (!F_ISSET(txn->mt_flags, MDB_TXN_FINISHED)) {
 		pgno_t *pghead = env->me_pghead;
 

From d7db12ad7796bf34f3b296ad3d6416e55af29a9e Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sun, 19 Jul 2015 21:43:10 +0200
Subject: [PATCH 105/504] ITS#7377 Always notice env error on txn startup.

Move the check to the end of txn startup.  Catches env
breakage which happens while the new txn waits for a lock.
---
 libraries/liblmdb/mdb.c | 23 +++++++++--------------
 1 file changed, 9 insertions(+), 14 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index b67bf09bec..532d94a7d7 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2743,12 +2743,16 @@ mdb_txn_renew0(MDB_txn *txn)
 	txn->mt_dbflags[MAIN_DBI] = DB_VALID|DB_USRVALID;
 	txn->mt_dbflags[FREE_DBI] = DB_VALID;
 
-	if (env->me_maxpg < txn->mt_next_pgno) {
-		mdb_txn_end(txn, new_notls /*0 or MDB_END_SLOT*/ | MDB_END_FAIL_BEGIN);
-		return MDB_MAP_RESIZED;
+	if (env->me_flags & MDB_FATAL_ERROR) {
+		DPUTS("environment had fatal error, must shutdown!");
+		rc = MDB_PANIC;
+	} else if (env->me_maxpg < txn->mt_next_pgno) {
+		rc = MDB_MAP_RESIZED;
+	} else {
+		return MDB_SUCCESS;
 	}
-
-	return MDB_SUCCESS;
+	mdb_txn_end(txn, new_notls /*0 or MDB_END_SLOT*/ | MDB_END_FAIL_BEGIN);
+	return rc;
 }
 
 int
@@ -2759,11 +2763,6 @@ mdb_txn_renew(MDB_txn *txn)
 	if (!txn || !F_ISSET(txn->mt_flags, MDB_TXN_RDONLY|MDB_TXN_FINISHED))
 		return EINVAL;
 
-	if (txn->mt_env->me_flags & MDB_FATAL_ERROR) {
-		DPUTS("environment had fatal error, must shutdown!");
-		return MDB_PANIC;
-	}
-
 	rc = mdb_txn_renew0(txn);
 	if (rc == MDB_SUCCESS) {
 		DPRINTF(("renew txn %"Z"u%c %p on mdbenv %p, root page %"Z"u",
@@ -2783,10 +2782,6 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 	flags &= MDB_TXN_BEGIN_FLAGS;
 	flags |= env->me_flags & MDB_WRITEMAP;
 
-	if (env->me_flags & MDB_FATAL_ERROR) {
-		DPUTS("environment had fatal error, must shutdown!");
-		return MDB_PANIC;
-	}
 	if (env->me_flags & MDB_RDONLY & ~flags) /* write txn in RDONLY env */
 		return EACCES;
 

From 4dea7d44fa612c74a3e521aae52228c5f2fc81f9 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sun, 19 Jul 2015 21:44:02 +0200
Subject: [PATCH 106/504] ITS#7377 Catch mdb_put() to blocked/read-only txns

...early enough that txn state is left unchanged.
---
 libraries/liblmdb/mdb.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 532d94a7d7..39d890e2d5 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -8602,6 +8602,9 @@ mdb_put(MDB_txn *txn, MDB_dbi dbi,
 	if (flags & ~(MDB_NOOVERWRITE|MDB_NODUPDATA|MDB_RESERVE|MDB_APPEND|MDB_APPENDDUP))
 		return EINVAL;
 
+	if (txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_BLOCKED))
+		return (txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN;
+
 	mdb_cursor_init(&mc, txn, dbi, &mx);
 	return mdb_cursor_put(&mc, key, data, flags);
 }

From af29f6a59418f72fb67c308a2785a6f8b07787fa Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sun, 26 Jul 2015 07:16:41 +0200
Subject: [PATCH 107/504] Kill magic numbers for NUM_METAS, databases

---
 libraries/liblmdb/mdb.c | 101 +++++++++++++++++++++-------------------
 1 file changed, 53 insertions(+), 48 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 39d890e2d5..cba7cfcbe6 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1004,6 +1004,11 @@ typedef struct MDB_db {
 #define	FREE_DBI	0
 	/** Handle for the default DB. */
 #define	MAIN_DBI	1
+	/** Number of DBs in metapage (free and main) - also hardcoded elsewhere */
+#define CORE_DBS	2
+
+	/** Number of meta pages - also hardcoded elsewhere */
+#define NUM_METAS	2
 
 	/** Meta page content.
 	 *	A meta page is the start point for accessing a database snapshot.
@@ -1017,11 +1022,11 @@ typedef struct MDB_meta {
 	uint32_t	mm_version;
 	void		*mm_address;		/**< address for fixed mapping */
 	size_t		mm_mapsize;			/**< size of mmap region */
-	MDB_db		mm_dbs[2];			/**< first is free space, 2nd is main db */
+	MDB_db		mm_dbs[CORE_DBS];	/**< first is free space, 2nd is main db */
 	/** The size of pages used in this DB */
-#define	mm_psize	mm_dbs[0].md_pad
+#define	mm_psize	mm_dbs[FREE_DBI].md_pad
 	/** Any persistent environment flags. @ref mdb_env */
-#define	mm_flags	mm_dbs[0].md_flags
+#define	mm_flags	mm_dbs[FREE_DBI].md_flags
 	pgno_t		mm_last_pg;			/**< last used page in file */
 	volatile txnid_t	mm_txnid;	/**< txnid that committed this page */
 } MDB_meta;
@@ -1238,7 +1243,7 @@ struct MDB_env {
 	char		*me_path;		/**< path to the DB files */
 	char		*me_map;		/**< the memory map of the data file */
 	MDB_txninfo	*me_txns;		/**< the memory map of the lock file or NULL */
-	MDB_meta	*me_metas[2];	/**< pointers to the two meta pages */
+	MDB_meta	*me_metas[NUM_METAS];	/**< pointers to the two meta pages */
 	void		*me_pbuf;		/**< scratch area for DUPSORT put() */
 	MDB_txn		*me_txn;		/**< current write transaction */
 	MDB_txn		*me_txn0;		/**< prealloc'd write transaction */
@@ -1686,9 +1691,10 @@ static void mdb_audit(MDB_txn *txn)
 			mdb_tassert(txn, rc == MDB_NOTFOUND);
 		}
 	}
-	if (freecount + count + 2 /* metapages */ != txn->mt_next_pgno) {
+	if (freecount + count + NUM_METAS != txn->mt_next_pgno) {
 		fprintf(stderr, "audit: %lu freecount: %lu count: %lu total: %lu next_pgno: %lu\n",
-			txn->mt_txnid, freecount, count+2, freecount+count+2, txn->mt_next_pgno);
+			txn->mt_txnid, freecount, count+NUM_METAS,
+			freecount+count+NUM_METAS, txn->mt_next_pgno);
 	}
 }
 #endif
@@ -1957,7 +1963,7 @@ mdb_page_spill(MDB_cursor *m0, MDB_val *key, MDB_val *data)
 	/* Estimate how much space this op will take */
 	i = m0->mc_db->md_depth;
 	/* Named DBs also dirty the main DB */
-	if (m0->mc_dbi > MAIN_DBI)
+	if (m0->mc_dbi >= CORE_DBS)
 		i += txn->mt_dbs[MAIN_DBI].md_depth;
 	/* For puts, roughly factor in the key+data size */
 	if (key)
@@ -2726,7 +2732,7 @@ mdb_txn_renew0(MDB_txn *txn)
 	}
 
 	/* Copy the DB info and flags */
-	memcpy(txn->mt_dbs, meta->mm_dbs, 2 * sizeof(MDB_db));
+	memcpy(txn->mt_dbs, meta->mm_dbs, CORE_DBS * sizeof(MDB_db));
 
 	/* Moved to here to avoid a data race in read TXNs */
 	txn->mt_next_pgno = meta->mm_last_pg+1;
@@ -2735,7 +2741,7 @@ mdb_txn_renew0(MDB_txn *txn)
 
 	/* Setup db info */
 	txn->mt_numdbs = env->me_numdbs;
-	for (i=2; i<txn->mt_numdbs; i++) {
+	for (i=CORE_DBS; i<txn->mt_numdbs; i++) {
 		x = env->me_dbflags[i];
 		txn->mt_dbs[i].md_flags = x & PERSISTENT_FLAGS;
 		txn->mt_dbflags[i] = (x & MDB_VALID) ? DB_VALID|DB_USRVALID|DB_STALE : 0;
@@ -2896,7 +2902,7 @@ mdb_dbis_update(MDB_txn *txn, int keep)
 	MDB_env *env = txn->mt_env;
 	unsigned char *tdbflags = txn->mt_dbflags;
 
-	for (i = n; --i >= 2;) {
+	for (i = n; --i >= CORE_DBS;) {
 		if (tdbflags[i] & DB_NEW) {
 			if (keep) {
 				env->me_dbflags[i] = txn->mt_dbs[i].md_flags | MDB_VALID;
@@ -3418,9 +3424,9 @@ mdb_txn_commit(MDB_txn *txn)
 		/* Update parent's DB table. */
 		memcpy(parent->mt_dbs, txn->mt_dbs, txn->mt_numdbs * sizeof(MDB_db));
 		parent->mt_numdbs = txn->mt_numdbs;
-		parent->mt_dbflags[0] = txn->mt_dbflags[0];
-		parent->mt_dbflags[1] = txn->mt_dbflags[1];
-		for (i=2; i<txn->mt_numdbs; i++) {
+		parent->mt_dbflags[FREE_DBI] = txn->mt_dbflags[FREE_DBI];
+		parent->mt_dbflags[MAIN_DBI] = txn->mt_dbflags[MAIN_DBI];
+		for (i=CORE_DBS; i<txn->mt_numdbs; i++) {
 			/* preserve parent's DB_NEW status */
 			x = parent->mt_dbflags[i] & DB_NEW;
 			parent->mt_dbflags[i] = txn->mt_dbflags[i] | x;
@@ -3521,14 +3527,14 @@ mdb_txn_commit(MDB_txn *txn)
 	    txn->mt_txnid, (void*)txn, (void*)env, txn->mt_dbs[MAIN_DBI].md_root));
 
 	/* Update DB root pointers */
-	if (txn->mt_numdbs > 2) {
+	if (txn->mt_numdbs > CORE_DBS) {
 		MDB_cursor mc;
 		MDB_dbi i;
 		MDB_val data;
 		data.mv_size = sizeof(MDB_db);
 
 		mdb_cursor_init(&mc, txn, MAIN_DBI, NULL);
-		for (i = 2; i < txn->mt_numdbs; i++) {
+		for (i = CORE_DBS; i < txn->mt_numdbs; i++) {
 			if (txn->mt_dbflags[i] & DB_DIRTY) {
 				if (TXN_DBI_CHANGED(txn, i)) {
 					rc = MDB_BAD_DBI;
@@ -3592,7 +3598,7 @@ mdb_env_read_header(MDB_env *env, MDB_meta *meta)
 	 * Read both meta pages so we can use the latest one.
 	 */
 
-	for (i=off=0; i<2; i++, off = meta->mm_psize) {
+	for (i=off=0; i<NUM_METAS; i++, off += meta->mm_psize) {
 #ifdef _WIN32
 		DWORD len;
 		OVERLAPPED ov;
@@ -3645,11 +3651,11 @@ mdb_env_init_meta0(MDB_env *env, MDB_meta *meta)
 	meta->mm_version = MDB_DATA_VERSION;
 	meta->mm_mapsize = env->me_mapsize;
 	meta->mm_psize = env->me_psize;
-	meta->mm_last_pg = 1;
+	meta->mm_last_pg = NUM_METAS-1;
 	meta->mm_flags = env->me_flags & 0xffff;
-	meta->mm_flags |= MDB_INTEGERKEY;
-	meta->mm_dbs[0].md_root = P_INVALID;
-	meta->mm_dbs[1].md_root = P_INVALID;
+	meta->mm_flags |= MDB_INTEGERKEY; /* this is mm_dbs[FREE_DBI].md_flags */
+	meta->mm_dbs[FREE_DBI].md_root = P_INVALID;
+	meta->mm_dbs[MAIN_DBI].md_root = P_INVALID;
 }
 
 /** Write the environment parameters of a freshly created DB environment.
@@ -3682,7 +3688,7 @@ mdb_env_init_meta(MDB_env *env, MDB_meta *meta)
 
 	psize = env->me_psize;
 
-	p = calloc(2, psize);
+	p = calloc(NUM_METAS, psize);
 	p->mp_pgno = 0;
 	p->mp_flags = P_META;
 	*(MDB_meta *)METADATA(p) = *meta;
@@ -3692,10 +3698,10 @@ mdb_env_init_meta(MDB_env *env, MDB_meta *meta)
 	q->mp_flags = P_META;
 	*(MDB_meta *)METADATA(q) = *meta;
 
-	DO_PWRITE(rc, env->me_fd, p, psize * 2, len, 0);
+	DO_PWRITE(rc, env->me_fd, p, psize * NUM_METAS, len, 0);
 	if (!rc)
 		rc = ErrCode();
-	else if ((unsigned) len == psize * 2)
+	else if ((unsigned) len == psize * NUM_METAS)
 		rc = MDB_SUCCESS;
 	else
 		rc = ENOSPC;
@@ -3738,8 +3744,8 @@ mdb_env_write_meta(MDB_txn *txn)
 
 	if (flags & MDB_WRITEMAP) {
 		mp->mm_mapsize = mapsize;
-		mp->mm_dbs[0] = txn->mt_dbs[0];
-		mp->mm_dbs[1] = txn->mt_dbs[1];
+		mp->mm_dbs[FREE_DBI] = txn->mt_dbs[FREE_DBI];
+		mp->mm_dbs[MAIN_DBI] = txn->mt_dbs[MAIN_DBI];
 		mp->mm_last_pg = txn->mt_next_pgno - 1;
 #if (__GNUC__ * 100 + __GNUC_MINOR__ >= 404) && /* TODO: portability */	\
 	!(defined(__i386__) || defined(__x86_64__))
@@ -3770,8 +3776,8 @@ mdb_env_write_meta(MDB_txn *txn)
 	metab.mm_last_pg = env->me_metas[toggle]->mm_last_pg;
 
 	meta.mm_mapsize = mapsize;
-	meta.mm_dbs[0] = txn->mt_dbs[0];
-	meta.mm_dbs[1] = txn->mt_dbs[1];
+	meta.mm_dbs[FREE_DBI] = txn->mt_dbs[FREE_DBI];
+	meta.mm_dbs[MAIN_DBI] = txn->mt_dbs[MAIN_DBI];
 	meta.mm_last_pg = txn->mt_next_pgno - 1;
 	meta.mm_txnid = txn->mt_txnid;
 
@@ -3854,7 +3860,7 @@ mdb_env_create(MDB_env **env)
 		return ENOMEM;
 
 	e->me_maxreaders = DEFAULT_READERS;
-	e->me_maxdbs = e->me_numdbs = 2;
+	e->me_maxdbs = e->me_numdbs = CORE_DBS;
 	e->me_fd = INVALID_HANDLE_VALUE;
 	e->me_lfd = INVALID_HANDLE_VALUE;
 	e->me_mfd = INVALID_HANDLE_VALUE;
@@ -3995,7 +4001,7 @@ mdb_env_set_maxdbs(MDB_env *env, MDB_dbi dbs)
 {
 	if (env->me_map)
 		return EINVAL;
-	env->me_maxdbs = dbs + 2; /* Named databases + main and free DB */
+	env->me_maxdbs = dbs + CORE_DBS;
 	return MDB_SUCCESS;
 }
 
@@ -4874,7 +4880,7 @@ mdb_env_close0(MDB_env *env, int excl)
 
 	/* Doing this here since me_dbxs may not exist during mdb_env_close */
 	if (env->me_dbxs) {
-		for (i = env->me_maxdbs; --i > MAIN_DBI; )
+		for (i = env->me_maxdbs; --i >= CORE_DBS; )
 			free(env->me_dbxs[i].md_name.mv_data);
 		free(env->me_dbxs);
 	}
@@ -6280,7 +6286,7 @@ mdb_cursor_touch(MDB_cursor *mc)
 {
 	int rc = MDB_SUCCESS;
 
-	if (mc->mc_dbi > MAIN_DBI && !(*mc->mc_dbflag & DB_DIRTY)) {
+	if (mc->mc_dbi >= CORE_DBS && !(*mc->mc_dbflag & DB_DIRTY)) {
 		MDB_cursor mc2;
 		MDB_xcursor mcx;
 		if (TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbi))
@@ -7354,8 +7360,7 @@ mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **ret)
 	if (txn->mt_flags & MDB_TXN_BLOCKED)
 		return MDB_BAD_TXN;
 
-	/* Allow read access to the freelist */
-	if (!dbi && !F_ISSET(txn->mt_flags, MDB_TXN_RDONLY))
+	if (dbi == FREE_DBI && !F_ISSET(txn->mt_flags, MDB_TXN_RDONLY))
 		return EINVAL;
 
 	if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT)
@@ -8909,7 +8914,7 @@ mdb_env_copyfd1(MDB_env *env, HANDLE fd)
 	my.mc_wlen[1] = 0;
 	my.mc_olen[0] = 0;
 	my.mc_olen[1] = 0;
-	my.mc_next_pgno = 2;
+	my.mc_next_pgno = NUM_METAS;
 	my.mc_status = 0;
 	my.mc_new = 1;
 	my.mc_toggle = 0;
@@ -8922,7 +8927,7 @@ mdb_env_copyfd1(MDB_env *env, HANDLE fd)
 		return rc;
 
 	mp = (MDB_page *)my.mc_wbuf[0];
-	memset(mp, 0, 2*env->me_psize);
+	memset(mp, 0, NUM_METAS * env->me_psize);
 	mp->mp_pgno = 0;
 	mp->mp_flags = P_META;
 	mm = (MDB_meta *)METADATA(mp);
@@ -8945,27 +8950,27 @@ mdb_env_copyfd1(MDB_env *env, HANDLE fd)
 		mdb_cursor_init(&mc, txn, FREE_DBI, NULL);
 		while ((rc = mdb_cursor_get(&mc, &key, &data, MDB_NEXT)) == 0)
 			freecount += *(MDB_ID *)data.mv_data;
-		freecount += txn->mt_dbs[0].md_branch_pages +
-			txn->mt_dbs[0].md_leaf_pages +
-			txn->mt_dbs[0].md_overflow_pages;
+		freecount += txn->mt_dbs[FREE_DBI].md_branch_pages +
+			txn->mt_dbs[FREE_DBI].md_leaf_pages +
+			txn->mt_dbs[FREE_DBI].md_overflow_pages;
 
 		/* Set metapage 1 */
 		mm->mm_last_pg = txn->mt_next_pgno - freecount - 1;
-		mm->mm_dbs[1] = txn->mt_dbs[1];
-		if (mm->mm_last_pg > 1) {
-			mm->mm_dbs[1].md_root = mm->mm_last_pg;
+		mm->mm_dbs[MAIN_DBI] = txn->mt_dbs[MAIN_DBI];
+		if (mm->mm_last_pg > NUM_METAS-1) {
+			mm->mm_dbs[MAIN_DBI].md_root = mm->mm_last_pg;
 			mm->mm_txnid = 1;
 		} else {
-			mm->mm_dbs[1].md_root = P_INVALID;
+			mm->mm_dbs[MAIN_DBI].md_root = P_INVALID;
 		}
 	}
-	my.mc_wlen[0] = env->me_psize * 2;
+	my.mc_wlen[0] = env->me_psize * NUM_METAS;
 	my.mc_txn = txn;
 	pthread_mutex_lock(&my.mc_mutex);
 	while(my.mc_new)
 		pthread_cond_wait(&my.mc_cond, &my.mc_mutex);
 	pthread_mutex_unlock(&my.mc_mutex);
-	rc = mdb_env_cwalk(&my, &txn->mt_dbs[1].md_root, 0);
+	rc = mdb_env_cwalk(&my, &txn->mt_dbs[MAIN_DBI].md_root, 0);
 	if (rc == MDB_SUCCESS && my.mc_wlen[my.mc_toggle])
 		rc = mdb_env_cthr_toggle(&my, 1);
 	mdb_env_cthr_toggle(&my, -1);
@@ -9029,7 +9034,7 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd)
 		}
 	}
 
-	wsize = env->me_psize * 2;
+	wsize = env->me_psize * NUM_METAS;
 	ptr = env->me_map;
 	w2 = wsize;
 	while (w2 > 0) {
@@ -9347,7 +9352,7 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db
 
 	/* Is the DB already open? */
 	len = strlen(name);
-	for (i=2; i<txn->mt_numdbs; i++) {
+	for (i=CORE_DBS; i<txn->mt_numdbs; i++) {
 		if (!txn->mt_dbxs[i].md_name.mv_size) {
 			/* Remember this free slot */
 			if (!unused) unused = i;
@@ -9436,7 +9441,7 @@ mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *arg)
 void mdb_dbi_close(MDB_env *env, MDB_dbi dbi)
 {
 	char *ptr;
-	if (dbi <= MAIN_DBI || dbi >= env->me_maxdbs)
+	if (dbi < CORE_DBS || dbi >= env->me_maxdbs)
 		return;
 	ptr = env->me_dbxs[dbi].md_name.mv_data;
 	/* If there was no name, this was already closed */
@@ -9573,7 +9578,7 @@ int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del)
 		goto leave;
 
 	/* Can't delete the main DB */
-	if (del && dbi > MAIN_DBI) {
+	if (del && dbi >= CORE_DBS) {
 		rc = mdb_del0(txn, MAIN_DBI, &mc->mc_dbx->md_name, NULL, F_SUBDATA);
 		if (!rc) {
 			txn->mt_dbflags[dbi] = DB_STALE;

From 2c6c267a2395a893c884cd61d988dd301e742a1b Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sun, 26 Jul 2015 07:20:19 +0200
Subject: [PATCH 108/504] Simpler mdb_env_write_meta()

---
 libraries/liblmdb/mdb.c | 19 +++++++------------
 1 file changed, 7 insertions(+), 12 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index cba7cfcbe6..80f6711434 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -3756,15 +3756,12 @@ mdb_env_write_meta(MDB_txn *txn)
 		if (!(flags & (MDB_NOMETASYNC|MDB_NOSYNC))) {
 			unsigned meta_size = env->me_psize;
 			rc = (env->me_flags & MDB_MAPASYNC) ? MS_ASYNC : MS_SYNC;
-			ptr = env->me_map;
-			if (toggle) {
+			ptr = (char *)mp - PAGEHDRSZ;
 #ifndef _WIN32	/* POSIX msync() requires ptr = start of OS page */
-				if (meta_size < env->me_os_psize)
-					meta_size += meta_size;
-				else
+			r2 = (ptr - env->me_map) & (env->me_os_psize - 1);
+			ptr -= r2;
+			meta_size += r2;
 #endif
-					ptr += meta_size;
-			}
 			if (MDB_MSYNC(ptr, meta_size, rc)) {
 				rc = ErrCode();
 				goto fail;
@@ -3772,8 +3769,8 @@ mdb_env_write_meta(MDB_txn *txn)
 		}
 		goto done;
 	}
-	metab.mm_txnid = env->me_metas[toggle]->mm_txnid;
-	metab.mm_last_pg = env->me_metas[toggle]->mm_last_pg;
+	metab.mm_txnid = mp->mm_txnid;
+	metab.mm_last_pg = mp->mm_last_pg;
 
 	meta.mm_mapsize = mapsize;
 	meta.mm_dbs[FREE_DBI] = txn->mt_dbs[FREE_DBI];
@@ -3784,9 +3781,7 @@ mdb_env_write_meta(MDB_txn *txn)
 	off = offsetof(MDB_meta, mm_mapsize);
 	ptr = (char *)&meta + off;
 	len = sizeof(MDB_meta) - off;
-	if (toggle)
-		off += env->me_psize;
-	off += PAGEHDRSZ;
+	off += (char *)mp - env->me_map;
 
 	/* Write to the SYNC fd */
 	mfd = (flags & (MDB_NOSYNC|MDB_NOMETASYNC)) ? env->me_fd : env->me_mfd;

From 572aff7e80809c1020ae92fd0a0d7b28c0b28825 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 29 Jul 2015 20:01:34 +0100
Subject: [PATCH 109/504] ITS#8192 fix reference to EINTR on WIN32 from
 ITS#8106

---
 libraries/liblmdb/mdb.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 80f6711434..6ccce2d3a2 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -3798,8 +3798,10 @@ retry_write:
 #endif
 	if (rc != len) {
 		rc = rc < 0 ? ErrCode() : EIO;
+#ifndef _WIN323
 		if (rc == EINTR)
 			goto retry_write;
+#endif
 		DPUTS("write failed, disk error?");
 		/* On a failure, the pagecache still contains the new data.
 		 * Write some old data back, to prevent it from being used.

From d4f9568249f51c65e7dc1e16e9645d326c57d032 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 29 Jul 2015 20:03:43 +0100
Subject: [PATCH 110/504] ITS#8192 cleanup unused label on WIN32

---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 6ccce2d3a2..12dad1f4d9 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -3785,7 +3785,6 @@ mdb_env_write_meta(MDB_txn *txn)
 
 	/* Write to the SYNC fd */
 	mfd = (flags & (MDB_NOSYNC|MDB_NOMETASYNC)) ? env->me_fd : env->me_mfd;
-retry_write:
 #ifdef _WIN32
 	{
 		memset(&ov, 0, sizeof(ov));
@@ -3794,6 +3793,7 @@ retry_write:
 			rc = -1;
 	}
 #else
+retry_write:
 	rc = pwrite(mfd, ptr, len, off);
 #endif
 	if (rc != len) {

From c9d45600cd181c27eb769c878ebe6a6a20fb3139 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 30 Jul 2015 00:54:05 +0100
Subject: [PATCH 111/504] ITS#8192 fix typo

---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 12dad1f4d9..61337a0a6b 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -3798,7 +3798,7 @@ retry_write:
 #endif
 	if (rc != len) {
 		rc = rc < 0 ? ErrCode() : EIO;
-#ifndef _WIN323
+#ifndef _WIN32
 		if (rc == EINTR)
 			goto retry_write;
 #endif

From 80e997a0ebeed252b8487e6983eb8ef9b76576fd Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Fri, 14 Aug 2015 01:01:40 +0100
Subject: [PATCH 112/504] Bump version to 0.9.16

---
 libraries/liblmdb/lmdb.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index b72418e4fb..801f6b0098 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -192,7 +192,7 @@ typedef int mdb_filehandle_t;
 /** Library minor version */
 #define MDB_VERSION_MINOR	9
 /** Library patch version */
-#define MDB_VERSION_PATCH	15
+#define MDB_VERSION_PATCH	16
 
 /** Combine args a,b,c into a single integer for easy version comparisons */
 #define MDB_VERINT(a,b,c)	(((a) << 24) | ((b) << 16) | (c))
@@ -202,7 +202,7 @@ typedef int mdb_filehandle_t;
 	MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH)
 
 /** The release date of this library version */
-#define MDB_VERSION_DATE	"June 19, 2015"
+#define MDB_VERSION_DATE	"August 14, 2015"
 
 /** A stringifier for the version info */
 #define MDB_VERSTR(a,b,c,d)	"LMDB " #a "." #b "." #c ": (" d ")"

From 8d198fdd5a0090e0158cd942109b0531a481dfc6 Mon Sep 17 00:00:00 2001
From: Pavel Medvedev <pmedvedev@gmail.com>
Date: Wed, 19 Aug 2015 16:39:08 +0100
Subject: [PATCH 113/504] ITS#8067 add ssize_t typedef for MSVC

---
 libraries/liblmdb/mdb.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 61337a0a6b..b5ecd21369 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -96,7 +96,13 @@ extern int cacheflush(char *addr, int nbytes, int cache);
 #include <stdlib.h>
 #include <string.h>
 #include <time.h>
+
+#ifdef _MSC_VER
+#include <io.h>
+typedef SSIZE_T	ssize_t;
+#else
 #include <unistd.h>
+#endif
 
 #if defined(__sun) || defined(ANDROID)
 /* Most platforms have posix_memalign, older may only have memalign */

From 10c808e4f9ef0d4049ccd92750f83c15aeff4bb8 Mon Sep 17 00:00:00 2001
From: Pavel Medvedev <pmedvedev@gmail.com>
Date: Wed, 19 Aug 2015 17:19:48 +0100
Subject: [PATCH 114/504] ITS#8069 char* strings on Windows are ASCII

---
 libraries/liblmdb/mdb.c | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index b5ecd21369..08aa9914ac 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1479,7 +1479,7 @@ mdb_strerror(int err)
 		;
 	}
 	buf[0] = 0;
-	FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM |
+	FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM |
 		FORMAT_MESSAGE_IGNORE_INSERTS,
 		NULL, err, 0, ptr, sizeof(buf), (va_list *)pad);
 	return ptr;
@@ -4479,7 +4479,7 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 	off_t size, rsize;
 
 #ifdef _WIN32
-	env->me_lfd = CreateFile(lpath, GENERIC_READ|GENERIC_WRITE,
+	env->me_lfd = CreateFileA(lpath, GENERIC_READ|GENERIC_WRITE,
 		FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, OPEN_ALWAYS,
 		FILE_ATTRIBUTE_NORMAL, NULL);
 #else
@@ -4583,9 +4583,9 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 		mdb_hash_enc(&val, encbuf);
 		sprintf(env->me_txns->mti_rmname, "Global\\MDBr%s", encbuf);
 		sprintf(env->me_txns->mti_wmname, "Global\\MDBw%s", encbuf);
-		env->me_rmutex = CreateMutex(&mdb_all_sa, FALSE, env->me_txns->mti_rmname);
+		env->me_rmutex = CreateMutexA(&mdb_all_sa, FALSE, env->me_txns->mti_rmname);
 		if (!env->me_rmutex) goto fail_errno;
-		env->me_wmutex = CreateMutex(&mdb_all_sa, FALSE, env->me_txns->mti_wmname);
+		env->me_wmutex = CreateMutexA(&mdb_all_sa, FALSE, env->me_txns->mti_wmname);
 		if (!env->me_wmutex) goto fail_errno;
 #elif defined(MDB_USE_POSIX_SEM)
 		struct stat stbuf;
@@ -4672,9 +4672,9 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 			goto fail;
 		}
 #ifdef _WIN32
-		env->me_rmutex = OpenMutex(SYNCHRONIZE, FALSE, env->me_txns->mti_rmname);
+		env->me_rmutex = OpenMutexA(SYNCHRONIZE, FALSE, env->me_txns->mti_rmname);
 		if (!env->me_rmutex) goto fail_errno;
-		env->me_wmutex = OpenMutex(SYNCHRONIZE, FALSE, env->me_txns->mti_wmname);
+		env->me_wmutex = OpenMutexA(SYNCHRONIZE, FALSE, env->me_txns->mti_wmname);
 		if (!env->me_wmutex) goto fail_errno;
 #elif defined(MDB_USE_POSIX_SEM)
 		env->me_rmutex = sem_open(env->me_txns->mti_rmname, 0);
@@ -4795,7 +4795,7 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 		len = OPEN_ALWAYS;
 	}
 	mode = FILE_ATTRIBUTE_NORMAL;
-	env->me_fd = CreateFile(dpath, oflags, FILE_SHARE_READ|FILE_SHARE_WRITE,
+	env->me_fd = CreateFileA(dpath, oflags, FILE_SHARE_READ|FILE_SHARE_WRITE,
 		NULL, len, mode, NULL);
 #else
 	if (F_ISSET(flags, MDB_RDONLY))
@@ -4825,7 +4825,7 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 			 */
 #ifdef _WIN32
 			len = OPEN_EXISTING;
-			env->me_mfd = CreateFile(dpath, oflags,
+			env->me_mfd = CreateFileA(dpath, oflags,
 				FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, len,
 				mode | FILE_FLAG_WRITE_THROUGH, NULL);
 #else
@@ -9134,7 +9134,7 @@ mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags)
 	 * already in the OS cache.
 	 */
 #ifdef _WIN32
-	newfd = CreateFile(lpath, GENERIC_WRITE, 0, NULL, CREATE_NEW,
+	newfd = CreateFileA(lpath, GENERIC_WRITE, 0, NULL, CREATE_NEW,
 				FILE_FLAG_NO_BUFFERING|FILE_FLAG_WRITE_THROUGH, NULL);
 #else
 	newfd = open(lpath, O_WRONLY|O_CREAT|O_EXCL, 0666);

From c77fbc6f35bc0f392f4944a5dbe498069428ddb4 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Fri, 21 Aug 2015 21:40:33 +0100
Subject: [PATCH 115/504] Another MDB_APPEND doc tweak

Missed this in 7ce29b9edbdaf34b7aeb545324008ed4dff62952
---
 libraries/liblmdb/lmdb.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 801f6b0098..e3d72dce00 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -1465,7 +1465,7 @@ int  mdb_cursor_get(MDB_cursor *cursor, MDB_val *key, MDB_val *data,
 	 *		database. No key comparisons are performed. This option allows
 	 *		fast bulk loading when keys are already known to be in the
 	 *		correct order. Loading unsorted keys with this flag will cause
-	 *		data corruption.
+	 *		a #MDB_KEYEXIST error.
 	 *	<li>#MDB_APPENDDUP - as above, but for sorted dup data.
 	 *	<li>#MDB_MULTIPLE - store multiple contiguous data elements in a
 	 *		single request. This flag may only be specified if the database

From bc4c177b9171947e35c431eded3510c9dba40357 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sat, 22 Aug 2015 14:00:12 +0100
Subject: [PATCH 116/504] Doc tweak - MDB_RESERVE / DUPSORT incompat

For those people who insist on ignoring the obvious.
---
 libraries/liblmdb/lmdb.h | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index e3d72dce00..e080e76d0f 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -1303,7 +1303,8 @@ int  mdb_get(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data);
 	 *		the next update operation or the transaction ends. This saves
 	 *		an extra memcpy if the data is being generated later.
 	 *		LMDB does nothing else with this memory, the caller is expected
-	 *		to modify all of the space requested.
+	 *		to modify all of the space requested. This flag must not be
+	 *		specified if the database was opened with #MDB_DUPSORT.
 	 *	<li>#MDB_APPEND - append the given key/data pair to the end of the
 	 *		database. This option allows fast bulk loading when keys are
 	 *		already known to be in the correct order. Loading unsorted keys
@@ -1460,7 +1461,8 @@ int  mdb_cursor_get(MDB_cursor *cursor, MDB_val *key, MDB_val *data,
 	 *	<li>#MDB_RESERVE - reserve space for data of the given size, but
 	 *		don't copy the given data. Instead, return a pointer to the
 	 *		reserved space, which the caller can fill in later. This saves
-	 *		an extra memcpy if the data is being generated later.
+	 *		an extra memcpy if the data is being generated later. This flag
+	 *		must not be specified if the database was opened with #MDB_DUPSORT.
 	 *	<li>#MDB_APPEND - append the given key/data pair to the end of the
 	 *		database. No key comparisons are performed. This option allows
 	 *		fast bulk loading when keys are already known to be in the

From e9e3c9fb514a9eaf6f8436e1683dca78c12b15de Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sun, 23 Aug 2015 11:12:45 +0200
Subject: [PATCH 117/504] ITS#7377 mdb_env_init_meta(): Catch calloc error

---
 libraries/liblmdb/mdb.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 08aa9914ac..5fc4fbcce6 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -3695,6 +3695,8 @@ mdb_env_init_meta(MDB_env *env, MDB_meta *meta)
 	psize = env->me_psize;
 
 	p = calloc(NUM_METAS, psize);
+	if (!p)
+		return ENOMEM;
 	p->mp_pgno = 0;
 	p->mp_flags = P_META;
 	*(MDB_meta *)METADATA(p) = *meta;

From 7e476e4983cfba45cefe793b8bd6e13c486b3989 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sun, 6 Sep 2015 19:18:19 +0100
Subject: [PATCH 118/504] ITS#8237 fix ITS#7589 regression

---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 5fc4fbcce6..4abf11a42b 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -8370,7 +8370,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 				psize = 0;
 				if (newindx <= split_indx || newindx >= nkeys) {
 					i = 0; j = 1;
-					k = newindx >= nkeys ? nkeys : split_indx+2;
+					k = newindx >= nkeys ? nkeys : split_indx+1+IS_LEAF(mp);
 				} else {
 					i = nkeys; j = -1;
 					k = split_indx-1;

From 62826b0965c4a2ee2e0d18c58210ae042424b7ce Mon Sep 17 00:00:00 2001
From: Luke Yeager <lyeager@nvidia.com>
Date: Fri, 18 Sep 2015 18:06:43 -0700
Subject: [PATCH 119/504] ITS#8256 Create install directories if needed

---
 libraries/liblmdb/Makefile | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile
index a8ea9706fa..e812840852 100644
--- a/libraries/liblmdb/Makefile
+++ b/libraries/liblmdb/Makefile
@@ -36,6 +36,10 @@ PROGS	= $(IPROGS) mtest mtest2 mtest3 mtest4 mtest5
 all:	$(ILIBS) $(PROGS)
 
 install: $(ILIBS) $(IPROGS) $(IHDRS)
+	mkdir -p $(DESTDIR)$(prefix)/bin
+	mkdir -p $(DESTDIR)$(prefix)/lib
+	mkdir -p $(DESTDIR)$(prefix)/include
+	mkdir -p $(DESTDIR)$(prefix)/man/man1
 	for f in $(IPROGS); do cp $$f $(DESTDIR)$(prefix)/bin; done
 	for f in $(ILIBS); do cp $$f $(DESTDIR)$(prefix)/lib; done
 	for f in $(IHDRS); do cp $$f $(DESTDIR)$(prefix)/include; done

From 7267e0f566ef87a95b22de6da277f3e350359ad0 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sat, 26 Sep 2015 16:50:28 +0100
Subject: [PATCH 120/504] ITS#8221 don't merge branch pages needlessly

---
 libraries/liblmdb/mdb.c | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 4abf11a42b..d0f1deca59 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7887,17 +7887,24 @@ mdb_rebalance(MDB_cursor *mc)
 {
 	MDB_node	*node;
 	int rc;
-	unsigned int ptop, minkeys;
+	unsigned int ptop, minkeys, thresh;
 	MDB_cursor	mn;
 	indx_t oldki;
 
+	if (IS_BRANCH(mc->mc_pg[mc->mc_top])) {
+		minkeys = 1;
+		thresh = 1;
+	} else {
+		minkeys = 2;
+		thresh = FILL_THRESHOLD;
+	}
 	minkeys = 1 + (IS_BRANCH(mc->mc_pg[mc->mc_top]));
 	DPRINTF(("rebalancing %s page %"Z"u (has %u keys, %.1f%% full)",
 	    IS_LEAF(mc->mc_pg[mc->mc_top]) ? "leaf" : "branch",
 	    mdb_dbg_pgno(mc->mc_pg[mc->mc_top]), NUMKEYS(mc->mc_pg[mc->mc_top]),
 		(float)PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) / 10));
 
-	if (PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) >= FILL_THRESHOLD &&
+	if (PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) >= thresh &&
 		NUMKEYS(mc->mc_pg[mc->mc_top]) >= minkeys) {
 		DPRINTF(("no need to rebalance page %"Z"u, above fill threshold",
 		    mdb_dbg_pgno(mc->mc_pg[mc->mc_top])));
@@ -8032,7 +8039,7 @@ mdb_rebalance(MDB_cursor *mc)
 	 * (A branch page must never have less than 2 keys.)
 	 */
 	minkeys = 1 + (IS_BRANCH(mn.mc_pg[mn.mc_top]));
-	if (PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) >= FILL_THRESHOLD && NUMKEYS(mn.mc_pg[mn.mc_top]) > minkeys) {
+	if (PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) >= thresh && NUMKEYS(mn.mc_pg[mn.mc_top]) > minkeys) {
 		rc = mdb_node_move(&mn, mc);
 		if (mc->mc_ki[ptop]) {
 			oldki++;

From 2247507bb8f57aea38e24ab236045544649624e0 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sun, 27 Sep 2015 22:34:22 +0100
Subject: [PATCH 121/504] ITS#8221 cleanup prev commit

---
 libraries/liblmdb/mdb.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index d0f1deca59..b5719e7e6c 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7898,7 +7898,6 @@ mdb_rebalance(MDB_cursor *mc)
 		minkeys = 2;
 		thresh = FILL_THRESHOLD;
 	}
-	minkeys = 1 + (IS_BRANCH(mc->mc_pg[mc->mc_top]));
 	DPRINTF(("rebalancing %s page %"Z"u (has %u keys, %.1f%% full)",
 	    IS_LEAF(mc->mc_pg[mc->mc_top]) ? "leaf" : "branch",
 	    mdb_dbg_pgno(mc->mc_pg[mc->mc_top]), NUMKEYS(mc->mc_pg[mc->mc_top]),
@@ -8038,7 +8037,6 @@ mdb_rebalance(MDB_cursor *mc)
 	 * move one key from it. Otherwise we should try to merge them.
 	 * (A branch page must never have less than 2 keys.)
 	 */
-	minkeys = 1 + (IS_BRANCH(mn.mc_pg[mn.mc_top]));
 	if (PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) >= thresh && NUMKEYS(mn.mc_pg[mn.mc_top]) > minkeys) {
 		rc = mdb_node_move(&mn, mc);
 		if (mc->mc_ki[ptop]) {

From 3cbbe3a0d8bcd846bdb896a6be997e1b000300aa Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 29 Sep 2015 17:31:56 +0100
Subject: [PATCH 122/504] ITS#8221 fix typos

---
 libraries/liblmdb/mdb.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index b5719e7e6c..c2da869ac3 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7892,10 +7892,10 @@ mdb_rebalance(MDB_cursor *mc)
 	indx_t oldki;
 
 	if (IS_BRANCH(mc->mc_pg[mc->mc_top])) {
-		minkeys = 1;
+		minkeys = 2;
 		thresh = 1;
 	} else {
-		minkeys = 2;
+		minkeys = 1;
 		thresh = FILL_THRESHOLD;
 	}
 	DPRINTF(("rebalancing %s page %"Z"u (has %u keys, %.1f%% full)",

From e034bd3d84661dbe4a083c530746f1b137cf2857 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 30 Sep 2015 18:11:20 +0100
Subject: [PATCH 123/504] ITS#8259 fix usage ordering

---
 libraries/liblmdb/mdb_dump.1 | 2 +-
 libraries/liblmdb/mdb_dump.c | 2 +-
 libraries/liblmdb/mdb_load.1 | 2 +-
 libraries/liblmdb/mdb_load.c | 2 +-
 libraries/liblmdb/mdb_stat.1 | 2 +-
 libraries/liblmdb/mdb_stat.c | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/libraries/liblmdb/mdb_dump.1 b/libraries/liblmdb/mdb_dump.1
index dd545049b2..5a647ba2ec 100644
--- a/libraries/liblmdb/mdb_dump.1
+++ b/libraries/liblmdb/mdb_dump.1
@@ -5,7 +5,6 @@
 mdb_dump \- LMDB environment export tool
 .SH SYNOPSIS
 .B mdb_dump
-.BR \ envpath
 [\c
 .BR \-V ]
 [\c
@@ -19,6 +18,7 @@ mdb_dump \- LMDB environment export tool
 [\c
 .BR \-a \ |
 .BI \-s \ subdb\fR]
+.BR \ envpath
 .SH DESCRIPTION
 The
 .B mdb_dump
diff --git a/libraries/liblmdb/mdb_dump.c b/libraries/liblmdb/mdb_dump.c
index 7202d865bc..16c0aae1b7 100644
--- a/libraries/liblmdb/mdb_dump.c
+++ b/libraries/liblmdb/mdb_dump.c
@@ -155,7 +155,7 @@ static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name)
 
 static void usage(char *prog)
 {
-	fprintf(stderr, "usage: %s dbpath [-V] [-f output] [-l] [-n] [-p] [-a|-s subdb]\n", prog);
+	fprintf(stderr, "usage: %s [-V] [-f output] [-l] [-n] [-p] [-a|-s subdb] dbpath\n", prog);
 	exit(EXIT_FAILURE);
 }
 
diff --git a/libraries/liblmdb/mdb_load.1 b/libraries/liblmdb/mdb_load.1
index 511ec552b4..712ed0540d 100644
--- a/libraries/liblmdb/mdb_load.1
+++ b/libraries/liblmdb/mdb_load.1
@@ -5,7 +5,6 @@
 mdb_load \- LMDB environment import tool
 .SH SYNOPSIS
 .B mdb_load
-.BR \ envpath
 [\c
 .BR \-V ]
 [\c
@@ -18,6 +17,7 @@ mdb_load \- LMDB environment import tool
 .BR \-N ]
 [\c
 .BR \-T ]
+.BR \ envpath
 .SH DESCRIPTION
 The
 .B mdb_load
diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c
index 1f6ce0b7ec..160bc1b885 100644
--- a/libraries/liblmdb/mdb_load.c
+++ b/libraries/liblmdb/mdb_load.c
@@ -276,7 +276,7 @@ badend:
 
 static void usage(void)
 {
-	fprintf(stderr, "usage: %s dbpath [-V] [-f input] [-n] [-s name] [-N] [-T]\n", prog);
+	fprintf(stderr, "usage: %s [-V] [-f input] [-n] [-s name] [-N] [-T] dbpath\n", prog);
 	exit(EXIT_FAILURE);
 }
 
diff --git a/libraries/liblmdb/mdb_stat.1 b/libraries/liblmdb/mdb_stat.1
index e6ee5ad585..351c017578 100644
--- a/libraries/liblmdb/mdb_stat.1
+++ b/libraries/liblmdb/mdb_stat.1
@@ -5,7 +5,6 @@
 mdb_stat \- LMDB environment status tool
 .SH SYNOPSIS
 .B mdb_stat
-.BR \ envpath
 [\c
 .BR \-V ]
 [\c
@@ -19,6 +18,7 @@ mdb_stat \- LMDB environment status tool
 [\c
 .BR \-a \ |
 .BI \-s \ subdb\fR]
+.BR \ envpath
 .SH DESCRIPTION
 The
 .B mdb_stat
diff --git a/libraries/liblmdb/mdb_stat.c b/libraries/liblmdb/mdb_stat.c
index 210609b324..6f4b3ee4f3 100644
--- a/libraries/liblmdb/mdb_stat.c
+++ b/libraries/liblmdb/mdb_stat.c
@@ -37,7 +37,7 @@ static void prstat(MDB_stat *ms)
 
 static void usage(char *prog)
 {
-	fprintf(stderr, "usage: %s dbpath [-V] [-n] [-e] [-r[r]] [-f[f[f]]] [-a|-s subdb]\n", prog);
+	fprintf(stderr, "usage: %s [-V] [-n] [-e] [-r[r]] [-f[f[f]]] [-a|-s subdb] dbpath\n", prog);
 	exit(EXIT_FAILURE);
 }
 

From 436333b16c54fdfa5dce0aeaa5a81eeb2f5ceb17 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Thu, 1 Oct 2015 19:43:58 +0200
Subject: [PATCH 124/504] Fix typo in mdb_tassert().

No effect on current code.
---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index c2da869ac3..85952a1281 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1491,7 +1491,7 @@ mdb_strerror(int err)
 /** assert(3) variant in cursor context */
 #define mdb_cassert(mc, expr)	mdb_assert0((mc)->mc_txn->mt_env, expr, #expr)
 /** assert(3) variant in transaction context */
-#define mdb_tassert(mc, expr)	mdb_assert0((txn)->mt_env, expr, #expr)
+#define mdb_tassert(txn, expr)	mdb_assert0((txn)->mt_env, expr, #expr)
 /** assert(3) variant in environment context */
 #define mdb_eassert(env, expr)	mdb_assert0(env, expr, #expr)
 

From 8b46dcc26d1e9897ab1da3a4a164cad4a4479a52 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sun, 4 Oct 2015 01:56:25 +0100
Subject: [PATCH 125/504] ITS#8258 fix rebalance/split

The tree height can also increase during rebalance, not just shrink.
This can happen if update_key needs to split a parent branch page.
---
 libraries/liblmdb/mdb.c | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 85952a1281..028f64646d 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7845,9 +7845,9 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst)
 		uint16_t depth = cdst->mc_db->md_depth;
 		mdb_cursor_pop(cdst);
 		rc = mdb_rebalance(cdst);
-		/* Did the tree shrink? */
-		if (depth > cdst->mc_db->md_depth)
-			snum--;
+		/* Did the tree height change? */
+		if (depth != cdst->mc_db->md_depth)
+			snum += cdst->mc_db->md_depth - depth;
 		cdst->mc_snum = snum;
 		cdst->mc_top = snum-1;
 	}
@@ -8039,7 +8039,7 @@ mdb_rebalance(MDB_cursor *mc)
 	 */
 	if (PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) >= thresh && NUMKEYS(mn.mc_pg[mn.mc_top]) > minkeys) {
 		rc = mdb_node_move(&mn, mc);
-		if (mc->mc_ki[ptop]) {
+		if (mc->mc_ki[mc->mc_top-1]) {
 			oldki++;
 		}
 	} else {
@@ -8253,8 +8253,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 		mc->mc_ki[0] = 0;
 		mc->mc_db->md_root = pp->mp_pgno;
 		DPRINTF(("root split! new root = %"Z"u", pp->mp_pgno));
-		mc->mc_db->md_depth++;
-		new_root = 1;
+		new_root = mc->mc_db->md_depth++;
 
 		/* Add left (implicit) pointer. */
 		if ((rc = mdb_node_add(mc, 0, NULL, NULL, mp->mp_pgno, 0)) != MDB_SUCCESS) {
@@ -8567,7 +8566,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 			if (new_root) {
 				int k;
 				/* root split */
-				for (k=m3->mc_top; k>=0; k--) {
+				for (k=new_root; k>=0; k--) {
 					m3->mc_ki[k+1] = m3->mc_ki[k];
 					m3->mc_pg[k+1] = m3->mc_pg[k];
 				}

From b42cd65193a85503a47a8afed2476c883d926b05 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 6 Oct 2015 07:57:14 +0100
Subject: [PATCH 126/504] ITS#8264 fix cursor_del cursor tracking

Some destination fixups need to happen immediately after nodes
are moved, before rebalancing
---
 libraries/liblmdb/mdb.c | 72 +++++++++++++++++++++++++----------------
 1 file changed, 45 insertions(+), 27 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 028f64646d..178b37301b 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7642,8 +7642,22 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst)
 		/* Adjust other cursors pointing to mp */
 		MDB_cursor *m2, *m3;
 		MDB_dbi dbi = csrc->mc_dbi;
-		MDB_page *mp = csrc->mc_pg[csrc->mc_top];
+		MDB_page *mp;
 
+		mp = cdst->mc_pg[csrc->mc_top];
+		for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) {
+			if (csrc->mc_flags & C_SUB)
+				m3 = &m2->mc_xcursor->mx_cursor;
+			else
+				m3 = m2;
+			if (m3 == cdst) continue;
+			if (m3->mc_pg[csrc->mc_top] == mp && m3->mc_ki[csrc->mc_top] >=
+				cdst->mc_ki[csrc->mc_top]) {
+				m3->mc_ki[csrc->mc_top]++;
+			}
+		}
+
+		mp = csrc->mc_pg[csrc->mc_top];
 		for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) {
 			if (csrc->mc_flags & C_SUB)
 				m3 = &m2->mc_xcursor->mx_cursor;
@@ -8079,37 +8093,16 @@ mdb_cursor_del0(MDB_cursor *mc)
 	MDB_page *mp;
 	indx_t ki;
 	unsigned int nkeys;
+	MDB_cursor *m2, *m3;
+	MDB_dbi dbi = mc->mc_dbi;
 
 	ki = mc->mc_ki[mc->mc_top];
+	mp = mc->mc_pg[mc->mc_top];
 	mdb_node_del(mc, mc->mc_db->md_pad);
 	mc->mc_db->md_entries--;
-	rc = mdb_rebalance(mc);
-
-	if (rc == MDB_SUCCESS) {
-		MDB_cursor *m2, *m3;
-		MDB_dbi dbi = mc->mc_dbi;
-
-		/* DB is totally empty now, just bail out.
-		 * Other cursors adjustments were already done
-		 * by mdb_rebalance and aren't needed here.
-		 */
-		if (!mc->mc_snum)
-			return rc;
-
-		mp = mc->mc_pg[mc->mc_top];
-		nkeys = NUMKEYS(mp);
-
-		/* if mc points past last node in page, find next sibling */
-		if (mc->mc_ki[mc->mc_top] >= nkeys) {
-			rc = mdb_cursor_sibling(mc, 1);
-			if (rc == MDB_NOTFOUND) {
-				mc->mc_flags |= C_EOF;
-				rc = MDB_SUCCESS;
-			}
-		}
-
+	{
 		/* Adjust other cursors pointing to mp */
-		for (m2 = mc->mc_txn->mt_cursors[dbi]; !rc && m2; m2=m2->mc_next) {
+		for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) {
 			m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2;
 			if (! (m2->mc_flags & m3->mc_flags & C_INITIALIZED))
 				continue;
@@ -8123,6 +8116,31 @@ mdb_cursor_del0(MDB_cursor *mc)
 					else if (mc->mc_db->md_flags & MDB_DUPSORT)
 						m3->mc_xcursor->mx_cursor.mc_flags |= C_EOF;
 				}
+			}
+		}
+	}
+	rc = mdb_rebalance(mc);
+
+	if (rc == MDB_SUCCESS) {
+		/* DB is totally empty now, just bail out.
+		 * Other cursors adjustments were already done
+		 * by mdb_rebalance and aren't needed here.
+		 */
+		if (!mc->mc_snum)
+			return rc;
+
+		mp = mc->mc_pg[mc->mc_top];
+		nkeys = NUMKEYS(mp);
+
+		/* Adjust other cursors pointing to mp */
+		for (m2 = mc->mc_txn->mt_cursors[dbi]; !rc && m2; m2=m2->mc_next) {
+			m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2;
+			if (! (m2->mc_flags & m3->mc_flags & C_INITIALIZED))
+				continue;
+			if (m3->mc_snum < mc->mc_snum)
+				continue;
+			if (m3->mc_pg[mc->mc_top] == mp) {
+				/* if m3 points past last node in page, find next sibling */
 				if (m3->mc_ki[mc->mc_top] >= nkeys) {
 					rc = mdb_cursor_sibling(m3, 1);
 					if (rc == MDB_NOTFOUND) {

From c456d219a9919e36605056d21929abb66788d4e7 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 7 Oct 2015 06:30:58 +0100
Subject: [PATCH 127/504] ITS#8263 fix cursor tracking in cursor_put

---
 libraries/liblmdb/mdb.c | 77 +++++++++++++++++++++++++++++++++--------
 1 file changed, 63 insertions(+), 14 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 178b37301b..0bbe544f10 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1384,6 +1384,7 @@ static int	mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data);
 static void	mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx);
 static void	mdb_xcursor_init0(MDB_cursor *mc);
 static void	mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node);
+static void	mdb_xcursor_init2(MDB_cursor *mc, MDB_node *node);
 
 static int	mdb_drop0(MDB_cursor *mc, int subs);
 static void mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi);
@@ -6765,23 +6766,23 @@ put_sub:
 				rc = mdb_cursor_put(&mc->mc_xcursor->mx_cursor, &dkey, &xdata, xflags);
 				if (rc)
 					goto bad_sub;
-				{
-					/* Adjust other cursors pointing to mp */
-					MDB_cursor *m2;
-					unsigned i = mc->mc_top;
-					MDB_page *mp = mc->mc_pg[i];
-
-					for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) {
-						if (m2 == mc || m2->mc_snum < mc->mc_snum) continue;
-						if (!(m2->mc_flags & C_INITIALIZED)) continue;
-						if (m2->mc_pg[i] == mp && m2->mc_ki[i] == mc->mc_ki[i]) {
-							mdb_xcursor_init1(m2, leaf);
-						}
-					}
-				}
 				/* we've done our job */
 				dkey.mv_size = 0;
 			}
+			{
+				/* Adjust other cursors pointing to mp */
+				MDB_cursor *m2;
+				unsigned i = mc->mc_top;
+				MDB_page *mp = mc->mc_pg[i];
+
+				for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) {
+					if (m2 == mc || m2->mc_snum < mc->mc_snum) continue;
+					if (!(m2->mc_flags & C_INITIALIZED)) continue;
+					if (m2->mc_pg[i] == mp && m2->mc_ki[i] == mc->mc_ki[i]) {
+						mdb_xcursor_init2(m2, leaf);
+					}
+				}
+			}
 			ecount = mc->mc_xcursor->mx_db.md_entries;
 			if (flags & MDB_APPENDDUP)
 				xflags |= MDB_APPEND;
@@ -7325,6 +7326,54 @@ mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node)
 #endif
 }
 
+
+/** Fixup a sorted-dups cursor due to underlying update.
+ *	Sets up some fields that depend on the data from the main cursor.
+ *	Almost the same as init1, but skips initialization steps if the
+ *	xcursor had already been used.
+ * @param[in] mc The main cursor whose sorted-dups cursor is to be fixed up.
+ * @param[in] node The data containing the #MDB_db record for the
+ * sorted-dup database.
+ */
+static void
+mdb_xcursor_init2(MDB_cursor *mc, MDB_node *node)
+{
+	MDB_xcursor *mx = mc->mc_xcursor;
+
+	if (node->mn_flags & F_SUBDATA) {
+		memcpy(&mx->mx_db, NODEDATA(node), sizeof(MDB_db));
+		mdb_page_get(mc->mc_txn,mx->mx_db.md_root,&mx->mx_cursor.mc_pg[0],NULL);
+	} else {
+		MDB_page *fp = NODEDATA(node);
+		mx->mx_db.md_entries = NUMKEYS(fp);
+		COPY_PGNO(mx->mx_db.md_root, fp->mp_pgno);
+		mx->mx_cursor.mc_pg[0] = fp;
+	}
+	if (!(mx->mx_cursor.mc_flags & C_INITIALIZED)) {
+		mx->mx_cursor.mc_snum = 1;
+		mx->mx_cursor.mc_top = 0;
+		mx->mx_cursor.mc_flags |= C_INITIALIZED;
+		mx->mx_cursor.mc_ki[0] = 0;
+		if (!(node->mn_flags & F_SUBDATA)) {
+			mx->mx_db.md_pad = 0;
+			mx->mx_db.md_flags = 0;
+			mx->mx_db.md_depth = 1;
+			mx->mx_db.md_branch_pages = 0;
+			mx->mx_db.md_leaf_pages = 1;
+			mx->mx_db.md_overflow_pages = 0;
+			if (mc->mc_db->md_flags & MDB_DUPFIXED) {
+				mx->mx_db.md_flags = MDB_DUPFIXED;
+				mx->mx_db.md_pad = mx->mx_cursor.mc_pg[0]->mp_pad;
+				if (mc->mc_db->md_flags & MDB_INTEGERDUP)
+					mx->mx_db.md_flags |= MDB_INTEGERKEY;
+			}
+		}
+	}
+	DPRINTF(("Sub-db -%u root page %"Z"u", mx->mx_cursor.mc_dbi,
+		mx->mx_db.md_root));
+	mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DIRTY; /* DB_DIRTY guides mdb_cursor_touch */
+}
+
 /** Initialize a cursor for a given transaction and database. */
 static void
 mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx)

From e211b870d0c49629260c2e421b58d7b2d7dfc30f Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sun, 11 Oct 2015 21:40:49 +0100
Subject: [PATCH 128/504] ITS#8263 streamline prev patch

---
 libraries/liblmdb/mdb.c | 55 +++++++++++++++++------------------------
 1 file changed, 22 insertions(+), 33 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 0bbe544f10..3c1e03eb53 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1384,7 +1384,7 @@ static int	mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data);
 static void	mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx);
 static void	mdb_xcursor_init0(MDB_cursor *mc);
 static void	mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node);
-static void	mdb_xcursor_init2(MDB_cursor *mc, MDB_node *node);
+static void	mdb_xcursor_init2(MDB_cursor *mc, MDB_xcursor *src_mx, int force);
 
 static int	mdb_drop0(MDB_cursor *mc, int subs);
 static void mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi);
@@ -6322,7 +6322,7 @@ mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data,
 {
 	MDB_env		*env;
 	MDB_node	*leaf = NULL;
-	MDB_page	*fp, *mp;
+	MDB_page	*fp, *mp, *sub_root = NULL;
 	uint16_t	fp_flags;
 	MDB_val		xdata, *rdata, dkey, olddata;
 	MDB_db dummy;
@@ -6602,6 +6602,7 @@ prep_subDB:
 					offset = env->me_psize - olddata.mv_size;
 					flags |= F_DUPDATA|F_SUBDATA;
 					dummy.md_root = mp->mp_pgno;
+					sub_root = mp;
 			}
 			if (mp != fp) {
 				mp->mp_flags = fp_flags | P_DIRTY;
@@ -6748,7 +6749,7 @@ new_sub:
 		 * DB are all zero size.
 		 */
 		if (do_sub) {
-			int xflags;
+			int xflags, new_dupdata;
 			size_t ecount;
 put_sub:
 			xdata.mv_size = 0;
@@ -6761,6 +6762,9 @@ put_sub:
 				xflags = (flags & MDB_NODUPDATA) ?
 					MDB_NOOVERWRITE|MDB_NOSPILL : MDB_NOSPILL;
 			}
+			if (sub_root)
+				mc->mc_xcursor->mx_cursor.mc_pg[0] = sub_root;
+			new_dupdata = (int)dkey.mv_size;
 			/* converted, write the original data first */
 			if (dkey.mv_size) {
 				rc = mdb_cursor_put(&mc->mc_xcursor->mx_cursor, &dkey, &xdata, xflags);
@@ -6769,9 +6773,10 @@ put_sub:
 				/* we've done our job */
 				dkey.mv_size = 0;
 			}
-			{
+			if (!(leaf->mn_flags & F_SUBDATA) || sub_root) {
 				/* Adjust other cursors pointing to mp */
 				MDB_cursor *m2;
+				MDB_xcursor *mx = mc->mc_xcursor;
 				unsigned i = mc->mc_top;
 				MDB_page *mp = mc->mc_pg[i];
 
@@ -6779,7 +6784,7 @@ put_sub:
 					if (m2 == mc || m2->mc_snum < mc->mc_snum) continue;
 					if (!(m2->mc_flags & C_INITIALIZED)) continue;
 					if (m2->mc_pg[i] == mp && m2->mc_ki[i] == mc->mc_ki[i]) {
-						mdb_xcursor_init2(m2, leaf);
+						mdb_xcursor_init2(m2, mx, new_dupdata);
 					}
 				}
 			}
@@ -7332,46 +7337,30 @@ mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node)
  *	Almost the same as init1, but skips initialization steps if the
  *	xcursor had already been used.
  * @param[in] mc The main cursor whose sorted-dups cursor is to be fixed up.
- * @param[in] node The data containing the #MDB_db record for the
- * sorted-dup database.
+ * @param[in] src_mx The xcursor of an up-to-date cursor.
+ * @param[in] new_dupdata True if converting from a non-#F_DUPDATA item.
  */
 static void
-mdb_xcursor_init2(MDB_cursor *mc, MDB_node *node)
+mdb_xcursor_init2(MDB_cursor *mc, MDB_xcursor *src_mx, int new_dupdata)
 {
 	MDB_xcursor *mx = mc->mc_xcursor;
 
-	if (node->mn_flags & F_SUBDATA) {
-		memcpy(&mx->mx_db, NODEDATA(node), sizeof(MDB_db));
-		mdb_page_get(mc->mc_txn,mx->mx_db.md_root,&mx->mx_cursor.mc_pg[0],NULL);
-	} else {
-		MDB_page *fp = NODEDATA(node);
-		mx->mx_db.md_entries = NUMKEYS(fp);
-		COPY_PGNO(mx->mx_db.md_root, fp->mp_pgno);
-		mx->mx_cursor.mc_pg[0] = fp;
-	}
-	if (!(mx->mx_cursor.mc_flags & C_INITIALIZED)) {
+	if (new_dupdata) {
 		mx->mx_cursor.mc_snum = 1;
 		mx->mx_cursor.mc_top = 0;
 		mx->mx_cursor.mc_flags |= C_INITIALIZED;
 		mx->mx_cursor.mc_ki[0] = 0;
-		if (!(node->mn_flags & F_SUBDATA)) {
-			mx->mx_db.md_pad = 0;
-			mx->mx_db.md_flags = 0;
-			mx->mx_db.md_depth = 1;
-			mx->mx_db.md_branch_pages = 0;
-			mx->mx_db.md_leaf_pages = 1;
-			mx->mx_db.md_overflow_pages = 0;
-			if (mc->mc_db->md_flags & MDB_DUPFIXED) {
-				mx->mx_db.md_flags = MDB_DUPFIXED;
-				mx->mx_db.md_pad = mx->mx_cursor.mc_pg[0]->mp_pad;
-				if (mc->mc_db->md_flags & MDB_INTEGERDUP)
-					mx->mx_db.md_flags |= MDB_INTEGERKEY;
-			}
-		}
+		mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DIRTY; /* DB_DIRTY guides mdb_cursor_touch */
+#if UINT_MAX < SIZE_MAX
+		mx->mx_dbx.md_cmp = src_mx->mx_dbx.md_cmp;
+#endif
+	} else if (!(mx->mx_cursor.mc_flags & C_INITIALIZED)) {
+		return;
 	}
+	mx->mx_db = src_mx->mx_db;
+	mx->mx_cursor.mc_pg[0] = src_mx->mx_cursor.mc_pg[0];
 	DPRINTF(("Sub-db -%u root page %"Z"u", mx->mx_cursor.mc_dbi,
 		mx->mx_db.md_root));
-	mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DIRTY; /* DB_DIRTY guides mdb_cursor_touch */
 }
 
 /** Initialize a cursor for a given transaction and database. */

From 7c1baed4debbf66d6ec13c4361780631eefe1bd9 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 12 Oct 2015 00:08:41 +0100
Subject: [PATCH 129/504] ITS#7771 fix cursor tracking on fake pages

node_del shifts nodes around, cursors pointing at fake pages
need to have their mc_pg[0] corrected.
---
 libraries/liblmdb/mdb.c | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 3c1e03eb53..83bd5fbdf6 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -6783,8 +6783,14 @@ put_sub:
 				for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) {
 					if (m2 == mc || m2->mc_snum < mc->mc_snum) continue;
 					if (!(m2->mc_flags & C_INITIALIZED)) continue;
-					if (m2->mc_pg[i] == mp && m2->mc_ki[i] == mc->mc_ki[i]) {
-						mdb_xcursor_init2(m2, mx, new_dupdata);
+					if (m2->mc_pg[i] == mp) {
+						if (m2->mc_ki[i] == mc->mc_ki[i]) {
+							mdb_xcursor_init2(m2, mx, new_dupdata);
+						} else if (!insert_key) {
+							MDB_node *n2 = NODEPTR(mp, m2->mc_ki[i]);
+							if (!(n2->mn_flags & F_SUBDATA))
+								m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(n2);
+						}
 					}
 				}
 			}
@@ -6882,12 +6888,18 @@ mdb_cursor_del(MDB_cursor *mc, unsigned int flags)
 					mdb_node_shrink(mp, mc->mc_ki[mc->mc_top]);
 					leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]);
 					mc->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf);
-					/* fix other sub-DB cursors pointed at this fake page */
+					/* fix other sub-DB cursors pointed at fake pages on this page */
 					for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) {
 						if (m2 == mc || m2->mc_snum < mc->mc_snum) continue;
-						if (m2->mc_pg[mc->mc_top] == mp &&
-							m2->mc_ki[mc->mc_top] == mc->mc_ki[mc->mc_top])
-							m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf);
+						if (m2->mc_pg[mc->mc_top] == mp) {
+							if (m2->mc_ki[mc->mc_top] == mc->mc_ki[mc->mc_top]) {
+								m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf);
+							} else {
+								MDB_node *n2 = NODEPTR(mp, m2->mc_ki[mc->mc_top]);
+								if (!(n2->mn_flags & F_SUBDATA))
+									m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(n2);
+							}
+						}
 					}
 				}
 				mc->mc_db->md_entries--;

From 9411de60d2b4a99e8d68519f3f9083f23b4280c5 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 12 Oct 2015 10:29:09 +0100
Subject: [PATCH 130/504] ITS#7771 more for prev commit

---
 libraries/liblmdb/mdb.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 83bd5fbdf6..8f7f3b39b7 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -6891,6 +6891,7 @@ mdb_cursor_del(MDB_cursor *mc, unsigned int flags)
 					/* fix other sub-DB cursors pointed at fake pages on this page */
 					for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) {
 						if (m2 == mc || m2->mc_snum < mc->mc_snum) continue;
+						if (!(m2->mc_flags & C_INITIALIZED)) continue;
 						if (m2->mc_pg[mc->mc_top] == mp) {
 							if (m2->mc_ki[mc->mc_top] == mc->mc_ki[mc->mc_top]) {
 								m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf);

From 619ccdd3da6a7ed2e12dac91e9dd012618e1c99e Mon Sep 17 00:00:00 2001
From: Ignacio Casal Quinteiro <ignacio.casal@nice-software.com>
Date: Thu, 15 Oct 2015 13:22:25 -0700
Subject: [PATCH 131/504] ITS#8270 win32: fix conversion error with MSVC

---
 libraries/liblmdb/mdb.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 8f7f3b39b7..901f6c40d3 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -210,6 +210,12 @@ union semun {
 #define ESECT
 #endif
 
+#ifdef _MSC_VER
+#define CALL_CONV WINAPI
+#else
+#define CALL_CONV
+#endif
+
 /** @defgroup internal	LMDB Internals
  *	@{
  */
@@ -8715,7 +8721,7 @@ typedef struct mdb_copy {
 } mdb_copy;
 
 	/** Dedicated writer thread for compacting copy. */
-static THREAD_RET ESECT
+static THREAD_RET ESECT CALL_CONV
 mdb_env_copythr(void *arg)
 {
 	mdb_copy *my = arg;

From 2ee9c4a30e4d3ce203efcd3f1726f515f45913cc Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Wed, 28 Oct 2015 09:37:23 +0100
Subject: [PATCH 132/504] Fix MDB_NOSYNC,MDB_NOMETASYNC thinko

---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 901f6c40d3..d4888beb74 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -3750,7 +3750,7 @@ mdb_env_write_meta(MDB_txn *txn)
 		toggle, txn->mt_dbs[MAIN_DBI].md_root));
 
 	env = txn->mt_env;
-	flags = txn->mt_flags & env->me_flags;
+	flags = txn->mt_flags | env->me_flags;
 	mp = env->me_metas[toggle];
 	mapsize = env->me_metas[toggle ^ 1]->mm_mapsize;
 	/* Persist any increases of mapsize config */

From 2be67975340d0d12b29d23f539a641928e5c3b95 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sat, 31 Oct 2015 02:11:47 +0000
Subject: [PATCH 133/504] Add MDB_NO_ROBUST to disable Robust Mutexes

---
 libraries/liblmdb/Makefile |  1 +
 libraries/liblmdb/mdb.c    | 20 +++++++++++++++++++-
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile
index e812840852..e8e55e66fb 100644
--- a/libraries/liblmdb/Makefile
+++ b/libraries/liblmdb/Makefile
@@ -13,6 +13,7 @@
 # - MDB_FDATASYNC
 # - MDB_FDATASYNC_WORKS
 # - MDB_USE_PWRITEV
+# - MDB_NO_ROBUST
 #
 # There may be other macros in mdb.c of interest. You should
 # read mdb.c before changing any of them.
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index d4888beb74..80b914b234 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -252,7 +252,25 @@ union semun {
 #define MDB_OWNERDEAD	EOWNERDEAD	/**< #LOCK_MUTEX0() result if dead owner */
 #endif
 
-#ifdef MDB_OWNERDEAD
+/* Android currently lacks Robust Mutex support */
+#if defined(ANDROID) && defined(MDB_USE_POSIX_MUTEX)
+#define MDB_NO_ROBUST	1
+#endif
+
+/** Some platforms define the EOWNERDEAD error code
+ * even though they don't support Robust Mutexes.
+ * Compile with -DMDB_NO_ROBUST, or use some other
+ * mechanism like -DMDB_USE_SYSV_SEM instead of
+ * -DMDB_USE_POSIX_MUTEX. (SysV semaphores are
+ * also Robust, but some systems don't support them
+ * either.)
+ */
+
+#ifndef MDB_NO_ROBUST
+#define MDB_NO_ROBUST	0
+#endif
+
+#if defined(MDB_OWNERDEAD) && !(MDB_NO_ROBUST)
 #define MDB_ROBUST_SUPPORTED	1
 #endif
 

From b9130d3d6325e0e4cc4fcb11bb266368652d4f80 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sat, 31 Oct 2015 02:27:31 +0000
Subject: [PATCH 134/504] Use MDB_USE_ROBUST not MDB_NO_ROBUST

---
 libraries/liblmdb/Makefile |  2 +-
 libraries/liblmdb/mdb.c    | 18 +++++++++---------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile
index e8e55e66fb..2c3817f049 100644
--- a/libraries/liblmdb/Makefile
+++ b/libraries/liblmdb/Makefile
@@ -13,7 +13,7 @@
 # - MDB_FDATASYNC
 # - MDB_FDATASYNC_WORKS
 # - MDB_USE_PWRITEV
-# - MDB_NO_ROBUST
+# - MDB_USE_ROBUST
 #
 # There may be other macros in mdb.c of interest. You should
 # read mdb.c before changing any of them.
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 80b914b234..1e5c0006d7 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -252,25 +252,25 @@ union semun {
 #define MDB_OWNERDEAD	EOWNERDEAD	/**< #LOCK_MUTEX0() result if dead owner */
 #endif
 
-/* Android currently lacks Robust Mutex support */
-#if defined(ANDROID) && defined(MDB_USE_POSIX_MUTEX)
-#define MDB_NO_ROBUST	1
-#endif
 
 /** Some platforms define the EOWNERDEAD error code
  * even though they don't support Robust Mutexes.
- * Compile with -DMDB_NO_ROBUST, or use some other
+ * Compile with -DMDB_USE_ROBUST=0, or use some other
  * mechanism like -DMDB_USE_SYSV_SEM instead of
  * -DMDB_USE_POSIX_MUTEX. (SysV semaphores are
  * also Robust, but some systems don't support them
  * either.)
  */
-
-#ifndef MDB_NO_ROBUST
-#define MDB_NO_ROBUST	0
+#ifndef MDB_USE_ROBUST
+/* Android currently lacks Robust Mutex support */
+#if defined(ANDROID) && defined(MDB_USE_POSIX_MUTEX) && !defined(MDB_USE_ROBUST)
+#define MDB_USE_ROBUST	0
+#else
+#define MDB_USE_ROBUST	1
 #endif
+#endif /* MDB_USE_ROBUST */
 
-#if defined(MDB_OWNERDEAD) && !(MDB_NO_ROBUST)
+#if defined(MDB_OWNERDEAD) && MDB_USE_ROBUST
 #define MDB_ROBUST_SUPPORTED	1
 #endif
 

From d372a2a199c49082540863eb11173999a98c3f4f Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 4 Nov 2015 17:03:48 +0000
Subject: [PATCH 135/504] ITS#8299 fix page_merge cursor fixup

The parent's mc_ki has changed. We need to fix that up as well
in other cursors.
---
 libraries/liblmdb/mdb.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 1e5c0006d7..353ff4ac49 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7915,6 +7915,7 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst)
 		/* Adjust other cursors pointing to mp */
 		MDB_cursor *m2, *m3;
 		MDB_dbi dbi = csrc->mc_dbi;
+		unsigned int top = csrc->mc_top;
 
 		for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) {
 			if (csrc->mc_flags & C_SUB)
@@ -7923,9 +7924,10 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst)
 				m3 = m2;
 			if (m3 == csrc) continue;
 			if (m3->mc_snum < csrc->mc_snum) continue;
-			if (m3->mc_pg[csrc->mc_top] == psrc) {
-				m3->mc_pg[csrc->mc_top] = pdst;
-				m3->mc_ki[csrc->mc_top] += nkeys;
+			if (m3->mc_pg[top] == psrc) {
+				m3->mc_pg[top] = pdst;
+				m3->mc_ki[top] += nkeys;
+				m3->mc_ki[top-1] = cdst->mc_ki[top-1];
 			}
 		}
 	}

From eddd9b753715765e5315b703ad9d3e3357dd00a9 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 4 Nov 2015 18:11:12 +0000
Subject: [PATCH 136/504] ITS#8238 fix DUPFIXED page_split

Parent mc_ki wasn't adjusted if new_indx was > split point
---
 libraries/liblmdb/mdb.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 353ff4ac49..317ba5e04e 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -8419,6 +8419,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 				rp->mp_upper -= ksize - sizeof(indx_t);
 				mc->mc_ki[mc->mc_top] = x;
 				mc->mc_pg[mc->mc_top] = rp;
+				mc->mc_ki[ptop]++;
 			}
 		} else {
 			int psize, nsize, k;

From 0c777dfc9949bb97e3897003e070bc61e569a63e Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 4 Nov 2015 20:38:30 +0000
Subject: [PATCH 137/504] ITS#8300 fix rebalance after node_move

ITS#8258, ITS#7829 fixes checked parent index to see if we were moving
from a left neighbor. Should have just checked to see if current index
was 0, meaning we added on the left. (Parent index may not tell us
anything meaningful after a nested rebalance.)
---
 libraries/liblmdb/mdb.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 317ba5e04e..eecb547781 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -8130,7 +8130,8 @@ mdb_rebalance(MDB_cursor *mc)
 	 */
 	if (PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) >= thresh && NUMKEYS(mn.mc_pg[mn.mc_top]) > minkeys) {
 		rc = mdb_node_move(&mn, mc);
-		if (mc->mc_ki[mc->mc_top-1]) {
+		if (!mc->mc_ki[mc->mc_top]) {
+			/* if we inserted on left, bump position up */
 			oldki++;
 		}
 	} else {

From 7a5a78ef0c1f8b8e6dfd9cb1aab97c6f1994c0b0 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 4 Nov 2015 21:01:30 +0000
Subject: [PATCH 138/504] ITS#8300 fix node_move

Don't adjust other cursors when we added a node on the right.
---
 libraries/liblmdb/mdb.c | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index eecb547781..0054f62b5a 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7719,16 +7719,19 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst)
 		MDB_dbi dbi = csrc->mc_dbi;
 		MDB_page *mp;
 
-		mp = cdst->mc_pg[csrc->mc_top];
-		for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) {
-			if (csrc->mc_flags & C_SUB)
-				m3 = &m2->mc_xcursor->mx_cursor;
-			else
-				m3 = m2;
-			if (m3 == cdst) continue;
-			if (m3->mc_pg[csrc->mc_top] == mp && m3->mc_ki[csrc->mc_top] >=
-				cdst->mc_ki[csrc->mc_top]) {
-				m3->mc_ki[csrc->mc_top]++;
+		/* If we're adding on the left, bump others up */
+		if (!cdst->mc_ki[csrc->mc_top]) {
+			mp = cdst->mc_pg[csrc->mc_top];
+			for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) {
+				if (csrc->mc_flags & C_SUB)
+					m3 = &m2->mc_xcursor->mx_cursor;
+				else
+					m3 = m2;
+				if (m3 == cdst) continue;
+				if (m3->mc_pg[csrc->mc_top] == mp && m3->mc_ki[csrc->mc_top] >=
+					cdst->mc_ki[csrc->mc_top]) {
+					m3->mc_ki[csrc->mc_top]++;
+				}
 			}
 		}
 

From b12bdaaade614996c9fcd36400775b40f4fe671c Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 4 Nov 2015 22:31:01 +0000
Subject: [PATCH 139/504] ITS#8300 more for node_move fixups

When moving a node from the right neighbor, a different
adjustment is needed.
---
 libraries/liblmdb/mdb.c | 43 +++++++++++++++++++++++++++++------------
 1 file changed, 31 insertions(+), 12 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 0054f62b5a..2617513f94 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7733,19 +7733,38 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst)
 					m3->mc_ki[csrc->mc_top]++;
 				}
 			}
-		}
 
-		mp = csrc->mc_pg[csrc->mc_top];
-		for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) {
-			if (csrc->mc_flags & C_SUB)
-				m3 = &m2->mc_xcursor->mx_cursor;
-			else
-				m3 = m2;
-			if (m3 == csrc) continue;
-			if (m3->mc_pg[csrc->mc_top] == mp && m3->mc_ki[csrc->mc_top] ==
-				csrc->mc_ki[csrc->mc_top]) {
-				m3->mc_pg[csrc->mc_top] = cdst->mc_pg[cdst->mc_top];
-				m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top];
+			mp = csrc->mc_pg[csrc->mc_top];
+			for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) {
+				if (csrc->mc_flags & C_SUB)
+					m3 = &m2->mc_xcursor->mx_cursor;
+				else
+					m3 = m2;
+				if (m3 == csrc) continue;
+				if (m3->mc_pg[csrc->mc_top] == mp && m3->mc_ki[csrc->mc_top] ==
+					csrc->mc_ki[csrc->mc_top]) {
+					m3->mc_pg[csrc->mc_top] = cdst->mc_pg[cdst->mc_top];
+					m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top];
+				}
+			}
+		} else
+		/* Adding on the right, bump others down */
+		{
+			mp = csrc->mc_pg[csrc->mc_top];
+			for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) {
+				if (csrc->mc_flags & C_SUB)
+					m3 = &m2->mc_xcursor->mx_cursor;
+				else
+					m3 = m2;
+				if (m3 == csrc) continue;
+				if (m3->mc_pg[csrc->mc_top] == mp) {
+					if (!m3->mc_ki[csrc->mc_top]) {
+						m3->mc_pg[csrc->mc_top] = cdst->mc_pg[cdst->mc_top];
+						m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top];
+					} else {
+						m3->mc_ki[csrc->mc_top]--;
+					}
+				}
 			}
 		}
 	}

From 3ff8a737044441c4a3553347fe830e470097050c Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 4 Nov 2015 22:45:38 +0000
Subject: [PATCH 140/504] ITS#8300 simplify

---
 libraries/liblmdb/mdb.c | 28 ++++++++++------------------
 1 file changed, 10 insertions(+), 18 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 2617513f94..a4d1f6146f 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7717,32 +7717,25 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst)
 		/* Adjust other cursors pointing to mp */
 		MDB_cursor *m2, *m3;
 		MDB_dbi dbi = csrc->mc_dbi;
-		MDB_page *mp;
+		MDB_page *mpd, *mps;
 
+		mps = csrc->mc_pg[csrc->mc_top];
 		/* If we're adding on the left, bump others up */
 		if (!cdst->mc_ki[csrc->mc_top]) {
-			mp = cdst->mc_pg[csrc->mc_top];
+			mpd = cdst->mc_pg[csrc->mc_top];
 			for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) {
 				if (csrc->mc_flags & C_SUB)
 					m3 = &m2->mc_xcursor->mx_cursor;
 				else
 					m3 = m2;
-				if (m3 == cdst) continue;
-				if (m3->mc_pg[csrc->mc_top] == mp && m3->mc_ki[csrc->mc_top] >=
-					cdst->mc_ki[csrc->mc_top]) {
+				if (m3 != cdst &&
+					m3->mc_pg[csrc->mc_top] == mpd &&
+					m3->mc_ki[csrc->mc_top] >= cdst->mc_ki[csrc->mc_top]) {
 					m3->mc_ki[csrc->mc_top]++;
 				}
-			}
-
-			mp = csrc->mc_pg[csrc->mc_top];
-			for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) {
-				if (csrc->mc_flags & C_SUB)
-					m3 = &m2->mc_xcursor->mx_cursor;
-				else
-					m3 = m2;
-				if (m3 == csrc) continue;
-				if (m3->mc_pg[csrc->mc_top] == mp && m3->mc_ki[csrc->mc_top] ==
-					csrc->mc_ki[csrc->mc_top]) {
+				if (m3 !=csrc &&
+					m3->mc_pg[csrc->mc_top] == mps &&
+					m3->mc_ki[csrc->mc_top] == csrc->mc_ki[csrc->mc_top]) {
 					m3->mc_pg[csrc->mc_top] = cdst->mc_pg[cdst->mc_top];
 					m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top];
 				}
@@ -7750,14 +7743,13 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst)
 		} else
 		/* Adding on the right, bump others down */
 		{
-			mp = csrc->mc_pg[csrc->mc_top];
 			for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) {
 				if (csrc->mc_flags & C_SUB)
 					m3 = &m2->mc_xcursor->mx_cursor;
 				else
 					m3 = m2;
 				if (m3 == csrc) continue;
-				if (m3->mc_pg[csrc->mc_top] == mp) {
+				if (m3->mc_pg[csrc->mc_top] == mps) {
 					if (!m3->mc_ki[csrc->mc_top]) {
 						m3->mc_pg[csrc->mc_top] = cdst->mc_pg[cdst->mc_top];
 						m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top];

From 0fdfdf320c5e043f424ea744e7a162cb932bc480 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Fri, 6 Nov 2015 10:12:45 +0000
Subject: [PATCH 141/504] ITS#8300 more for node_move

When moving a node from one page to another, make sure other cursors'
parent index gets adjusted too.
---
 libraries/liblmdb/mdb.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index a4d1f6146f..59222cadea 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7738,6 +7738,7 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst)
 					m3->mc_ki[csrc->mc_top] == csrc->mc_ki[csrc->mc_top]) {
 					m3->mc_pg[csrc->mc_top] = cdst->mc_pg[cdst->mc_top];
 					m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top];
+					m3->mc_ki[csrc->mc_top-1]++;
 				}
 			}
 		} else
@@ -7753,6 +7754,7 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst)
 					if (!m3->mc_ki[csrc->mc_top]) {
 						m3->mc_pg[csrc->mc_top] = cdst->mc_pg[cdst->mc_top];
 						m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top];
+						m3->mc_ki[csrc->mc_top-1]--;
 					} else {
 						m3->mc_ki[csrc->mc_top]--;
 					}

From 3c9786b5e8dae8a6e8e49a83c55824eb8e4f2cea Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 9 Nov 2015 20:09:10 +0000
Subject: [PATCH 142/504] ITS#8304 fix page_merge

Was using the pre-touch dst page pointer instead of the touched page
---
 libraries/liblmdb/mdb.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 59222cadea..6d9e0a4415 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7854,6 +7854,9 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst)
 	if ((rc = mdb_page_touch(cdst)))
 		return rc;
 
+	/* get dst page again now that we've touched it. */
+	pdst = cdst->mc_pg[cdst->mc_top];
+
 	/* Move all nodes from src to dst.
 	 */
 	j = nkeys = NUMKEYS(pdst);

From c6d908075ee5bdd1270fce6ec8d5d8df9b168021 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 11 Nov 2015 00:02:06 +0000
Subject: [PATCH 143/504] Tweak robust mutex detection for glibc

---
 libraries/liblmdb/mdb.c | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 6d9e0a4415..c7580a58c3 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -252,7 +252,9 @@ union semun {
 #define MDB_OWNERDEAD	EOWNERDEAD	/**< #LOCK_MUTEX0() result if dead owner */
 #endif
 
-
+#ifdef __GLIBC__
+#define	GLIBC_VER	((__GLIBC__ << 16 )| __GLIBC_MINOR__)
+#endif
 /** Some platforms define the EOWNERDEAD error code
  * even though they don't support Robust Mutexes.
  * Compile with -DMDB_USE_ROBUST=0, or use some other
@@ -262,12 +264,19 @@ union semun {
  * either.)
  */
 #ifndef MDB_USE_ROBUST
-/* Android currently lacks Robust Mutex support */
-#if defined(ANDROID) && defined(MDB_USE_POSIX_MUTEX) && !defined(MDB_USE_ROBUST)
-#define MDB_USE_ROBUST	0
-#else
-#define MDB_USE_ROBUST	1
-#endif
+/* Android currently lacks Robust Mutex support. So does glibc < 2.4. */
+# if defined(MDB_USE_POSIX_MUTEX) && (defined(ANDROID) || \
+	(defined(__GLIBC__) && GLIBC_VER < 0x020004))
+#  define MDB_USE_ROBUST	0
+# else
+#  define MDB_USE_ROBUST	1
+/* glibc < 2.10 only provided _np API */
+#  if defined(__GLIBC__) && GLIBC_VER < 0x02000a
+#   define PTHREAD_MUTEX_ROBUST	PTHREAD_MUTEX_ROBUST_NP
+#   define pthread_mutexattr_setrobust(attr, flag)	pthread_mutexattr_setrobust_np(attr, flag)
+#   define pthread_mutex_consistent(mutex)	pthread_mutex_consistent_np(mutex)
+#  endif
+# endif
 #endif /* MDB_USE_ROBUST */
 
 #if defined(MDB_OWNERDEAD) && MDB_USE_ROBUST

From c46119a545410f282b524c297d43929ccca80361 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Fri, 13 Nov 2015 09:25:55 +0000
Subject: [PATCH 144/504] ITS#8310 fix mdb_rebalance cursor fixup

When collapsing root, must adjust entire cursor stack
---
 libraries/liblmdb/mdb.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index c7580a58c3..6c8f0cef32 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -8091,9 +8091,9 @@ mdb_rebalance(MDB_cursor *mc)
 						m3 = &m2->mc_xcursor->mx_cursor;
 					else
 						m3 = m2;
-					if (m3 == mc || m3->mc_snum < mc->mc_snum) continue;
+					if (m3 == mc) continue;
 					if (m3->mc_pg[0] == mp) {
-						for (i=0; i<m3->mc_snum; i++) {
+						for (i=0; i<mc->mc_db->md_depth; i++) {
 							m3->mc_pg[i] = m3->mc_pg[i+1];
 							m3->mc_ki[i] = m3->mc_ki[i+1];
 						}

From 3360cbad668f678fb23c064ca4efcc5c9ae95d10 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 17 Nov 2015 18:22:25 +0100
Subject: [PATCH 145/504] ITS#7209 Default MDB_DSYNC = O_SYNC if no O_DSYNC

---
 libraries/liblmdb/mdb.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 6c8f0cef32..927c52c65c 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -459,10 +459,13 @@ static int mdb_mutex_failed(MDB_env *env, mdb_mutexref_t mutex, int rc);
  *
  *	@note If O_DSYNC is undefined but exists in /usr/include,
  * preferably set some compiler flag to get the definition.
- * Otherwise compile with the less efficient -DMDB_DSYNC=O_SYNC.
  */
 #ifndef MDB_DSYNC
+# ifdef O_DSYNC
 # define MDB_DSYNC	O_DSYNC
+# else
+# define MDB_DSYNC	O_SYNC
+# endif
 #endif
 #endif
 

From 6d63afb1d9194cf1c0408b865f78e94efbf40990 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 18 Nov 2015 00:33:25 +0000
Subject: [PATCH 146/504] ITS#8311 fix page_split from update_key

Check for top of stack. Usually the cursor only has height 1 when
calling page_split, but not always.
---
 libraries/liblmdb/mdb.c | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 927c52c65c..f3c87bf5c8 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -6512,16 +6512,18 @@ fix_parent:
 			 * update branch key if there is a parent page
 			 */
 			if (mc->mc_top && !mc->mc_ki[mc->mc_top]) {
-				unsigned short top = mc->mc_top;
+				unsigned short dtop = 1;
 				mc->mc_top--;
 				/* slot 0 is always an empty key, find real slot */
-				while (mc->mc_top && !mc->mc_ki[mc->mc_top])
+				while (mc->mc_top && !mc->mc_ki[mc->mc_top]) {
 					mc->mc_top--;
+					dtop++;
+				}
 				if (mc->mc_ki[mc->mc_top])
 					rc2 = mdb_update_key(mc, key);
 				else
 					rc2 = MDB_SUCCESS;
-				mc->mc_top = top;
+				mc->mc_top += dtop;
 				if (rc2)
 					return rc2;
 			}
@@ -8370,12 +8372,14 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 	rp->mp_pad = mp->mp_pad;
 	DPRINTF(("new right sibling: page %"Z"u", rp->mp_pgno));
 
-	if (mc->mc_snum < 2) {
+	if (mc->mc_top < 1) {
 		if ((rc = mdb_page_new(mc, P_BRANCH, 1, &pp)))
 			goto done;
 		/* shift current top to make room for new parent */
-		mc->mc_pg[1] = mc->mc_pg[0];
-		mc->mc_ki[1] = mc->mc_ki[0];
+		for (i=mc->mc_snum; i>0; i--) {
+			mc->mc_pg[i] = mc->mc_pg[i-1];
+			mc->mc_ki[i] = mc->mc_ki[i-1];
+		}
 		mc->mc_pg[0] = pp;
 		mc->mc_ki[0] = 0;
 		mc->mc_db->md_root = pp->mp_pgno;
@@ -8391,8 +8395,8 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 			mc->mc_db->md_depth--;
 			goto done;
 		}
-		mc->mc_snum = 2;
-		mc->mc_top = 1;
+		mc->mc_snum++;
+		mc->mc_top++;
 		ptop = 0;
 	} else {
 		ptop = mc->mc_top-1;

From 355f64ad07537a4f21dc6fe4cef66324316aa84b Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 18 Nov 2015 10:24:41 +0000
Subject: [PATCH 147/504] ITS#8311 add comment

---
 libraries/liblmdb/mdb.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index f3c87bf5c8..6bd025d2bf 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -8372,6 +8372,11 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 	rp->mp_pad = mp->mp_pad;
 	DPRINTF(("new right sibling: page %"Z"u", rp->mp_pgno));
 
+	/* Usually when splitting the root page, the cursor
+	 * height is 1. But when called from mdb_update_key,
+	 * the cursor height may be greater because it walks
+	 * up the stack while finding the branch slot to update.
+	 */
 	if (mc->mc_top < 1) {
 		if ((rc = mdb_page_new(mc, P_BRANCH, 1, &pp)))
 			goto done;

From 4d140b92ee3b12d293120c04e6b61e395bbec089 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Wed, 18 Nov 2015 16:30:24 +0100
Subject: [PATCH 148/504] ITS#8312 Fix loose pages in commit(nested txn)

---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 6bd025d2bf..a0ae06c729 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -3543,7 +3543,7 @@ mdb_txn_commit(MDB_txn *txn)
 		}
 
 		/* Append our loose page list to parent's */
-		for (lp = &parent->mt_loose_pgs; *lp; lp = &NEXT_LOOSE_PAGE(lp))
+		for (lp = &parent->mt_loose_pgs; *lp; lp = &NEXT_LOOSE_PAGE(*lp))
 			;
 		*lp = txn->mt_loose_pgs;
 		parent->mt_loose_count += txn->mt_loose_count;

From f08b70d0df323ccb33b94a648e9a1b6efd718cae Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 18 Nov 2015 21:33:51 +0000
Subject: [PATCH 149/504] ITS#8313 more for ITS#8062

dummy flags must be init'd due to 3d46d550
---
 libraries/liblmdb/mdb.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index a0ae06c729..9f299737f2 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -8176,6 +8176,7 @@ mdb_rebalance(MDB_cursor *mc)
 			mn.mc_ki[mn.mc_top] += mc->mc_ki[mn.mc_top] + 1;
 			/* We want mdb_rebalance to find mn when doing fixups */
 			if (mc->mc_flags & C_SUB) {
+				dummy.mc_flags = C_INITIALIZED;
 				dummy.mc_next = mc->mc_txn->mt_cursors[mc->mc_dbi];
 				mc->mc_txn->mt_cursors[mc->mc_dbi] = &dummy;
 				dummy.mc_xcursor = (MDB_xcursor *)&mn;

From a98b74e6df6fdb9dbc29d44fe2879f96191d8904 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 18 Nov 2015 23:38:34 +0000
Subject: [PATCH 150/504] ITS#8315 fix ovpage_free

Keep dirty_room sync'd with dirty_list
---
 libraries/liblmdb/mdb.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 9f299737f2..99da353d7d 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5570,6 +5570,7 @@ mdb_ovpage_free(MDB_cursor *mc, MDB_page *mp)
 				return MDB_CORRUPTED;
 			}
 		}
+		txn->mt_dirty_room++;
 		if (!(env->me_flags & MDB_WRITEMAP))
 			mdb_dpage_free(env, mp);
 release:
@@ -6704,6 +6705,7 @@ current:
 						return ENOMEM;
 					id2.mid = pg;
 					id2.mptr = np;
+					/* Note - this page is already counted in parent's dirty_room */
 					rc2 = mdb_mid2l_insert(mc->mc_txn->mt_u.dirty_list, &id2);
 					mdb_cassert(mc, rc2 == 0);
 					if (!(flags & MDB_RESERVE)) {

From 60c34d14ec3ceb0762d620eef77db73dd1e911dc Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 19 Nov 2015 20:04:16 +0000
Subject: [PATCH 151/504] ITS#8316 cursor fixup in page_merge

Deleting the merged page requires bumping down other ki's
of the page's parent.
---
 libraries/liblmdb/mdb.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 99da353d7d..9c0c95e91a 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7963,6 +7963,8 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst)
 				m3->mc_pg[top] = pdst;
 				m3->mc_ki[top] += nkeys;
 				m3->mc_ki[top-1] = cdst->mc_ki[top-1];
+			} else if (m3->mc_ki[top-1] > csrc->mc_ki[top-1]) {
+				m3->mc_ki[top-1]--;
 			}
 		}
 	}

From 58ad1dd757efc9c6854ddf67a3826f6d72ad2130 Mon Sep 17 00:00:00 2001
From: Oskari Timperi <oskari.timperi@iki.fi>
Date: Fri, 5 Dec 2014 12:56:22 +0200
Subject: [PATCH 152/504] ITS#7992 assume Windows paths are UTF-8

---
 libraries/liblmdb/mdb.c | 44 +++++++++++++++++++++++++++++++++++++----
 1 file changed, 40 insertions(+), 4 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 9c0c95e91a..74b7463df4 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1441,6 +1441,8 @@ static MDB_cmp_func	mdb_cmp_memn, mdb_cmp_memnr, mdb_cmp_int, mdb_cmp_cint, mdb_
 static SECURITY_DESCRIPTOR mdb_null_sd;
 static SECURITY_ATTRIBUTES mdb_all_sa;
 static int mdb_sec_inited;
+
+static int utf8_to_utf16(const char *src, int srcsize, wchar_t **dst, int *dstsize);
 #endif
 
 /** Return the library version info. */
@@ -4518,9 +4520,12 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 	off_t size, rsize;
 
 #ifdef _WIN32
-	env->me_lfd = CreateFileA(lpath, GENERIC_READ|GENERIC_WRITE,
+	wchar_t *wlpath;
+	utf8_to_utf16(lpath, -1, &wlpath, NULL);
+	env->me_lfd = CreateFileW(wlpath, GENERIC_READ|GENERIC_WRITE,
 		FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, OPEN_ALWAYS,
 		FILE_ATTRIBUTE_NORMAL, NULL);
+	free(wlpath);
 #else
 	env->me_lfd = open(lpath, O_RDWR|O_CREAT|MDB_CLOEXEC, mode);
 #endif
@@ -4771,6 +4776,9 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 {
 	int		oflags, rc, len, excl = -1;
 	char *lpath, *dpath;
+#ifdef _WIN32
+	wchar_t *wpath;
+#endif
 
 	if (env->me_fd!=INVALID_HANDLE_VALUE || (flags & ~(CHANGEABLE|CHANGELESS)))
 		return EINVAL;
@@ -4834,8 +4842,10 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 		len = OPEN_ALWAYS;
 	}
 	mode = FILE_ATTRIBUTE_NORMAL;
-	env->me_fd = CreateFileA(dpath, oflags, FILE_SHARE_READ|FILE_SHARE_WRITE,
+	utf8_to_utf16(dpath, -1, &wpath, NULL);
+	env->me_fd = CreateFileW(wpath, oflags, FILE_SHARE_READ|FILE_SHARE_WRITE,
 		NULL, len, mode, NULL);
+	free(wpath);
 #else
 	if (F_ISSET(flags, MDB_RDONLY))
 		oflags = O_RDONLY;
@@ -4864,9 +4874,11 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 			 */
 #ifdef _WIN32
 			len = OPEN_EXISTING;
-			env->me_mfd = CreateFileA(dpath, oflags,
+			utf8_to_utf16(dpath, -1, &wpath, NULL);
+			env->me_mfd = CreateFileW(wpath, oflags,
 				FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, len,
 				mode | FILE_FLAG_WRITE_THROUGH, NULL);
+			free(wpath);
 #else
 			oflags &= ~O_CREAT;
 			env->me_mfd = open(dpath, oflags | MDB_DSYNC, mode);
@@ -9265,6 +9277,9 @@ mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags)
 	int rc, len;
 	char *lpath;
 	HANDLE newfd = INVALID_HANDLE_VALUE;
+#ifdef _WIN32
+	wchar_t *wpath;
+#endif
 
 	if (env->me_flags & MDB_NOSUBDIR) {
 		lpath = (char *)path;
@@ -9282,8 +9297,10 @@ mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags)
 	 * already in the OS cache.
 	 */
 #ifdef _WIN32
-	newfd = CreateFileA(lpath, GENERIC_WRITE, 0, NULL, CREATE_NEW,
+	utf8_to_utf16(lpath, -1, &wpath, NULL);
+	newfd = CreateFileW(wpath, GENERIC_WRITE, 0, NULL, CREATE_NEW,
 				FILE_FLAG_NO_BUFFERING|FILE_FLAG_WRITE_THROUGH, NULL);
+	free(wpath);
 #else
 	newfd = open(lpath, O_WRONLY|O_CREAT|O_EXCL, 0666);
 #endif
@@ -9988,3 +10005,22 @@ mdb_mutex_failed(MDB_env *env, mdb_mutexref_t mutex, int rc)
 }
 #endif	/* MDB_ROBUST_SUPPORTED */
 /** @} */
+
+#if defined(_WIN32)
+static int utf8_to_utf16(const char *src, int srcsize, wchar_t **dst, int *dstsize)
+{
+	int need;
+	wchar_t *result;
+	need = MultiByteToWideChar(CP_UTF8, 0, src, srcsize, NULL, 0);
+	if (need == 0xFFFD)
+		return EILSEQ;
+	if (need == 0)
+		return EINVAL;
+	result = malloc(sizeof(wchar_t) * need);
+	MultiByteToWideChar(CP_UTF8, 0, src, srcsize, result, need);
+	if (dstsize)
+		*dstsize = need;
+	*dst = result;
+	return 0;
+}
+#endif /* defined(_WIN32) */

From 1bfe7578bdf425657f62557e8fac0ed7f925a0c9 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Fri, 20 Nov 2015 09:20:16 +0000
Subject: [PATCH 153/504] Refix root split check from
 5da67968afb599697d7557c13b65fb961ec408dd

---
 libraries/liblmdb/mdb.c | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 74b7463df4..a122c29cb9 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -8570,6 +8570,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 	/* Copy separator key to the parent.
 	 */
 	if (SIZELEFT(mn.mc_pg[ptop]) < mdb_branch_size(env, &sepkey)) {
+		int snum = mc->mc_snum;
 		mn.mc_snum--;
 		mn.mc_top--;
 		did_split = 1;
@@ -8578,13 +8579,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 			goto done;
 
 		/* root split? */
-		if (mn.mc_snum == mc->mc_snum) {
-			mc->mc_pg[mc->mc_snum] = mc->mc_pg[mc->mc_top];
-			mc->mc_ki[mc->mc_snum] = mc->mc_ki[mc->mc_top];
-			mc->mc_pg[mc->mc_top] = mc->mc_pg[ptop];
-			mc->mc_ki[mc->mc_top] = mc->mc_ki[ptop];
-			mc->mc_snum++;
-			mc->mc_top++;
+		if (mc->mc_snum > snum) {
 			ptop++;
 		}
 		/* Right page might now have changed parent.
@@ -8715,8 +8710,6 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 				continue;
 			if (!(m2->mc_flags & m3->mc_flags & C_INITIALIZED))
 				continue;
-			if (m3->mc_flags & C_SPLITTING)
-				continue;
 			if (new_root) {
 				int k;
 				/* root split */
@@ -8733,6 +8726,8 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 				m3->mc_snum++;
 				m3->mc_top++;
 			}
+			if (m3->mc_flags & C_SPLITTING)
+				continue;
 			if (m3->mc_top >= mc->mc_top && m3->mc_pg[mc->mc_top] == mp) {
 				if (m3->mc_ki[mc->mc_top] >= newindx && !(nflags & MDB_SPLIT_REPLACE))
 					m3->mc_ki[mc->mc_top]++;

From 946e4d3b21d1a48aa468d41663bbe8180284670a Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Fri, 20 Nov 2015 09:47:56 +0000
Subject: [PATCH 154/504] Silence some valgrind uninit warnings

---
 libraries/liblmdb/mdb.c | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index a122c29cb9..c6d21563ee 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7756,6 +7756,8 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst)
 					m3 = &m2->mc_xcursor->mx_cursor;
 				else
 					m3 = m2;
+				if (!(m3->mc_flags & C_INITIALIZED) || m3->mc_top < csrc->mc_top)
+					continue;
 				if (m3 != cdst &&
 					m3->mc_pg[csrc->mc_top] == mpd &&
 					m3->mc_ki[csrc->mc_top] >= cdst->mc_ki[csrc->mc_top]) {
@@ -7778,6 +7780,8 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst)
 				else
 					m3 = m2;
 				if (m3 == csrc) continue;
+				if (!(m3->mc_flags & C_INITIALIZED) || m3->mc_top < csrc->mc_top)
+					continue;
 				if (m3->mc_pg[csrc->mc_top] == mps) {
 					if (!m3->mc_ki[csrc->mc_top]) {
 						m3->mc_pg[csrc->mc_top] = cdst->mc_pg[cdst->mc_top];
@@ -8077,7 +8081,8 @@ mdb_rebalance(MDB_cursor *mc)
 						m3 = &m2->mc_xcursor->mx_cursor;
 					else
 						m3 = m2;
-					if (m3->mc_snum < mc->mc_snum) continue;
+					if (!(m3->mc_flags & C_INITIALIZED) || (m3->mc_snum < mc->mc_snum))
+						continue;
 					if (m3->mc_pg[0] == mp) {
 						m3->mc_snum = 0;
 						m3->mc_top = 0;
@@ -8113,6 +8118,8 @@ mdb_rebalance(MDB_cursor *mc)
 					else
 						m3 = m2;
 					if (m3 == mc) continue;
+					if (!(m3->mc_flags & C_INITIALIZED))
+						continue;
 					if (m3->mc_pg[0] == mp) {
 						for (i=0; i<mc->mc_db->md_depth; i++) {
 							m3->mc_pg[i] = m3->mc_pg[i+1];

From d6ae2553df446b1589299f6edfc6135ac70439b5 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Fri, 20 Nov 2015 13:34:11 +0000
Subject: [PATCH 155/504] ITS#8300 more for prev commit

Just tell explicitly which direction we moved/merged from
---
 libraries/liblmdb/mdb.c | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index c6d21563ee..7f896daa09 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1396,7 +1396,7 @@ static int  mdb_node_add(MDB_cursor *mc, indx_t indx,
 			    MDB_val *key, MDB_val *data, pgno_t pgno, unsigned int flags);
 static void mdb_node_del(MDB_cursor *mc, int ksize);
 static void mdb_node_shrink(MDB_page *mp, indx_t indx);
-static int	mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst);
+static int	mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft);
 static int  mdb_node_read(MDB_txn *txn, MDB_node *leaf, MDB_val *data);
 static size_t	mdb_leaf_size(MDB_env *env, MDB_val *key, MDB_val *data);
 static size_t	mdb_branch_size(MDB_env *env, MDB_val *key);
@@ -7647,7 +7647,7 @@ mdb_cursor_copy(const MDB_cursor *csrc, MDB_cursor *cdst);
 /** Move a node from csrc to cdst.
  */
 static int
-mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst)
+mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft)
 {
 	MDB_node		*srcnode;
 	MDB_val		 key, data;
@@ -7749,7 +7749,7 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst)
 
 		mps = csrc->mc_pg[csrc->mc_top];
 		/* If we're adding on the left, bump others up */
-		if (!cdst->mc_ki[csrc->mc_top]) {
+		if (fromleft) {
 			mpd = cdst->mc_pg[csrc->mc_top];
 			for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) {
 				if (csrc->mc_flags & C_SUB)
@@ -8030,7 +8030,7 @@ static int
 mdb_rebalance(MDB_cursor *mc)
 {
 	MDB_node	*node;
-	int rc;
+	int rc, fromleft;
 	unsigned int ptop, minkeys, thresh;
 	MDB_cursor	mn;
 	indx_t oldki;
@@ -8163,6 +8163,7 @@ mdb_rebalance(MDB_cursor *mc)
 			return rc;
 		mn.mc_ki[mn.mc_top] = 0;
 		mc->mc_ki[mc->mc_top] = NUMKEYS(mc->mc_pg[mc->mc_top]);
+		fromleft = 0;
 	} else {
 		/* There is at least one neighbor to the left.
 		 */
@@ -8174,6 +8175,7 @@ mdb_rebalance(MDB_cursor *mc)
 			return rc;
 		mn.mc_ki[mn.mc_top] = NUMKEYS(mn.mc_pg[mn.mc_top]) - 1;
 		mc->mc_ki[mc->mc_top] = 0;
+		fromleft = 1;
 	}
 
 	DPRINTF(("found neighbor page %"Z"u (%u keys, %.1f%% full)",
@@ -8185,13 +8187,13 @@ mdb_rebalance(MDB_cursor *mc)
 	 * (A branch page must never have less than 2 keys.)
 	 */
 	if (PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) >= thresh && NUMKEYS(mn.mc_pg[mn.mc_top]) > minkeys) {
-		rc = mdb_node_move(&mn, mc);
-		if (!mc->mc_ki[mc->mc_top]) {
+		rc = mdb_node_move(&mn, mc, fromleft);
+		if (fromleft) {
 			/* if we inserted on left, bump position up */
 			oldki++;
 		}
 	} else {
-		if (mc->mc_ki[ptop] == 0) {
+		if (!fromleft) {
 			rc = mdb_page_merge(&mn, mc);
 		} else {
 			MDB_cursor dummy;

From f3e38565e4af652b3da60e805ac465b76eb1d84b Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sat, 21 Nov 2015 17:58:49 +0000
Subject: [PATCH 156/504] ITS#8316 more for
 60c34d14ec3ceb0762d620eef77db73dd1e911dc

---
 libraries/liblmdb/mdb.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 7f896daa09..1f12db3947 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7979,7 +7979,8 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst)
 				m3->mc_pg[top] = pdst;
 				m3->mc_ki[top] += nkeys;
 				m3->mc_ki[top-1] = cdst->mc_ki[top-1];
-			} else if (m3->mc_ki[top-1] > csrc->mc_ki[top-1]) {
+			} else if (m3->mc_pg[top-1] == csrc->mc_pg[top-1] &&
+				m3->mc_ki[top-1] > csrc->mc_ki[top-1]) {
 				m3->mc_ki[top-1]--;
 			}
 		}

From f3eb26c9bb54e94a2ad553a6831433b0e1408e24 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sun, 22 Nov 2015 22:11:30 +0000
Subject: [PATCH 157/504] ITS#8321 Fix del/dupsort

When deleting a dupsort key, if other cursors pointed at that key,
set them to uninit'd, not EOF. They no longer have anything to
point at.
---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 1f12db3947..329bf8101e 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -8252,7 +8252,7 @@ mdb_cursor_del0(MDB_cursor *mc)
 					if (m3->mc_ki[mc->mc_top] > ki)
 						m3->mc_ki[mc->mc_top]--;
 					else if (mc->mc_db->md_flags & MDB_DUPSORT)
-						m3->mc_xcursor->mx_cursor.mc_flags |= C_EOF;
+						m3->mc_xcursor->mx_cursor.mc_flags &= ~C_INITIALIZED;
 				}
 			}
 		}

From 3b9f857d8ba37879141f612d96993e90426a7b0d Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 23 Nov 2015 01:07:57 +0000
Subject: [PATCH 158/504] ITS#8321 don't skip fixups on splitting cursors

Adjustments can't be skipped, in recursive calls each level must
fixup their own level.
---
 libraries/liblmdb/mdb.c | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 329bf8101e..451d8d3309 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1230,7 +1230,6 @@ struct MDB_cursor {
 #define C_EOF	0x02			/**< No more data */
 #define C_SUB	0x04			/**< Cursor is a sub-cursor */
 #define C_DEL	0x08			/**< last op was a cursor_del */
-#define C_SPLITTING	0x20		/**< Cursor is in page_split */
 #define C_UNTRACK	0x40		/**< Un-track cursor when closing */
 /** @} */
 	unsigned int	mc_flags;	/**< @ref mdb_cursor */
@@ -8435,7 +8434,6 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 		DPRINTF(("parent branch page is %"Z"u", mc->mc_pg[ptop]->mp_pgno));
 	}
 
-	mc->mc_flags |= C_SPLITTING;
 	mdb_cursor_copy(mc, &mn);
 	mn.mc_pg[mn.mc_top] = rp;
 	mn.mc_ki[ptop] = mc->mc_ki[ptop]+1;
@@ -8615,7 +8613,6 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 		rc = mdb_node_add(&mn, mn.mc_ki[ptop], &sepkey, NULL, rp->mp_pgno, 0);
 		mn.mc_top++;
 	}
-	mc->mc_flags ^= C_SPLITTING;
 	if (rc != MDB_SUCCESS) {
 		goto done;
 	}
@@ -8736,8 +8733,6 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 				m3->mc_snum++;
 				m3->mc_top++;
 			}
-			if (m3->mc_flags & C_SPLITTING)
-				continue;
 			if (m3->mc_top >= mc->mc_top && m3->mc_pg[mc->mc_top] == mp) {
 				if (m3->mc_ki[mc->mc_top] >= newindx && !(nflags & MDB_SPLIT_REPLACE))
 					m3->mc_ki[mc->mc_top]++;

From 88ac6987073a4d87e07b9a88f68c219e2649f0a5 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 23 Nov 2015 01:10:05 +0000
Subject: [PATCH 159/504] ITS#8321 fix mdb_cursor_chk()

It was reporting spurious errors due to uninit'd cursors
---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 451d8d3309..c4f3c450a0 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1673,7 +1673,7 @@ mdb_cursor_chk(MDB_cursor *mc)
 	MDB_node *node;
 	MDB_page *mp;
 
-	if (!mc->mc_snum && !(mc->mc_flags & C_INITIALIZED)) return;
+	if (!mc->mc_snum || !(mc->mc_flags & C_INITIALIZED)) return;
 	for (i=0; i<mc->mc_top; i++) {
 		mp = mc->mc_pg[i];
 		node = NODEPTR(mp, mc->mc_ki[i]);

From 0d5ebc508ecbd1f7d23b985257ec5993a764f17c Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 23 Nov 2015 01:13:16 +0000
Subject: [PATCH 160/504] ITS#8321 fix mdb_cursor_shadow()

Set a valid txn so that cursor fixup code works on the shadows
---
 libraries/liblmdb/mdb.c | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index c4f3c450a0..4598a54746 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2558,14 +2558,15 @@ mdb_cursor_shadow(MDB_txn *src, MDB_txn *dst)
 				*bk = *mc;
 				mc->mc_backup = bk;
 				mc->mc_db = &dst->mt_dbs[i];
-				/* Kill pointers into src - and dst to reduce abuse: The
-				 * user may not use mc until dst ends. Otherwise we'd...
+				/* Kill pointers into src to reduce abuse: The
+				 * user may not use mc until dst ends. But we need a valid
+				 * txn pointer here for cursor fixups to keep working.
 				 */
-				mc->mc_txn    = NULL;	/* ...set this to dst */
-				mc->mc_dbflag = NULL;	/* ...and &dst->mt_dbflags[i] */
+				mc->mc_txn    = dst;
+				mc->mc_dbflag = &dst->mt_dbflags[i];
 				if ((mx = mc->mc_xcursor) != NULL) {
 					*(MDB_xcursor *)(bk+1) = *mx;
-					mx->mx_cursor.mc_txn = NULL; /* ...and dst. */
+					mx->mx_cursor.mc_txn = dst;
 				}
 				mc->mc_next = dst->mt_cursors[i];
 				dst->mt_cursors[i] = mc;

From 828107eb634dbf9ee13f6966e2c78cb27259927f Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 23 Nov 2015 01:16:36 +0000
Subject: [PATCH 161/504] ITS#8321 fix mdb_cursor_put

Ignore sub-cursors that shouldn't be fixed up
---
 libraries/liblmdb/mdb.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 4598a54746..31cf188471 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -6831,6 +6831,7 @@ put_sub:
 				MDB_xcursor *mx = mc->mc_xcursor;
 				unsigned i = mc->mc_top;
 				MDB_page *mp = mc->mc_pg[i];
+				int nkeys = NUMKEYS(mp);
 
 				for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) {
 					if (m2 == mc || m2->mc_snum < mc->mc_snum) continue;
@@ -6838,9 +6839,9 @@ put_sub:
 					if (m2->mc_pg[i] == mp) {
 						if (m2->mc_ki[i] == mc->mc_ki[i]) {
 							mdb_xcursor_init2(m2, mx, new_dupdata);
-						} else if (!insert_key) {
+						} else if (!insert_key && m2->mc_ki[i] < nkeys) {
 							MDB_node *n2 = NODEPTR(mp, m2->mc_ki[i]);
-							if (!(n2->mn_flags & F_SUBDATA))
+							if ((n2->mn_flags & (F_SUBDATA|F_DUPDATA)) == F_DUPDATA)
 								m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(n2);
 						}
 					}

From 3a31dd5a8a6cad16ab12cbdb27d3a760be3dec81 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 23 Nov 2015 01:30:02 +0000
Subject: [PATCH 162/504] ITS#8321 track temporary cursors

In rebalance/split operations, temporary cursors need to be visible
to propagate fixups
---
 libraries/liblmdb/mdb.c | 51 ++++++++++++++++++++++++++++-------------
 1 file changed, 35 insertions(+), 16 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 31cf188471..7ac24610ff 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7645,6 +7645,22 @@ mdb_update_key(MDB_cursor *mc, MDB_val *key)
 static void
 mdb_cursor_copy(const MDB_cursor *csrc, MDB_cursor *cdst);
 
+/** Track a temporary cursor */
+#define CURSOR_TMP_TRACK(mc, mn, dummy, tracked) \
+	if (mc->mc_flags & C_SUB) { \
+		dummy.mc_flags =  C_INITIALIZED; \
+		dummy.mc_xcursor = (MDB_xcursor *)&mn; \
+		tracked = &dummy; \
+	} else { \
+		tracked = &mn; \
+	} \
+	tracked->mc_next = mc->mc_txn->mt_cursors[mc->mc_dbi]; \
+	mc->mc_txn->mt_cursors[mc->mc_dbi] = tracked
+
+/** Stop tracking a temporary cursor */
+#define CURSOR_TMP_UNTRACK(mc, tracked) \
+	mc->mc_txn->mt_cursors[mc->mc_dbi] = tracked->mc_next
+
 /** Move a node from csrc to cdst.
  */
 static int
@@ -7800,6 +7816,7 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft)
 	 */
 	if (csrc->mc_ki[csrc->mc_top] == 0) {
 		if (csrc->mc_ki[csrc->mc_top-1] != 0) {
+			MDB_cursor dummy, *tracked;
 			if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) {
 				key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], 0, key.mv_size);
 			} else {
@@ -7812,7 +7829,11 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft)
 			mdb_cursor_copy(csrc, &mn);
 			mn.mc_snum--;
 			mn.mc_top--;
-			if ((rc = mdb_update_key(&mn, &key)) != MDB_SUCCESS)
+			/* We want mdb_rebalance to find mn when doing fixups */
+			CURSOR_TMP_TRACK(csrc, mn, dummy, tracked);
+			rc = mdb_update_key(&mn, &key);
+			CURSOR_TMP_UNTRACK(csrc, tracked);
+			if (rc)
 				return rc;
 		}
 		if (IS_BRANCH(csrc->mc_pg[csrc->mc_top])) {
@@ -7828,6 +7849,7 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft)
 
 	if (cdst->mc_ki[cdst->mc_top] == 0) {
 		if (cdst->mc_ki[cdst->mc_top-1] != 0) {
+			MDB_cursor dummy, *tracked;
 			if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) {
 				key.mv_data = LEAF2KEY(cdst->mc_pg[cdst->mc_top], 0, key.mv_size);
 			} else {
@@ -7840,7 +7862,11 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft)
 			mdb_cursor_copy(cdst, &mn);
 			mn.mc_snum--;
 			mn.mc_top--;
-			if ((rc = mdb_update_key(&mn, &key)) != MDB_SUCCESS)
+			/* We want mdb_rebalance to find mn when doing fixups */
+			CURSOR_TMP_TRACK(cdst, mn, dummy, tracked);
+			rc = mdb_update_key(&mn, &key);
+			CURSOR_TMP_UNTRACK(cdst, tracked);
+			if (rc)
 				return rc;
 		}
 		if (IS_BRANCH(cdst->mc_pg[cdst->mc_top])) {
@@ -8198,24 +8224,13 @@ mdb_rebalance(MDB_cursor *mc)
 		if (!fromleft) {
 			rc = mdb_page_merge(&mn, mc);
 		} else {
-			MDB_cursor dummy;
+			MDB_cursor dummy, *tracked;
 			oldki += NUMKEYS(mn.mc_pg[mn.mc_top]);
 			mn.mc_ki[mn.mc_top] += mc->mc_ki[mn.mc_top] + 1;
 			/* We want mdb_rebalance to find mn when doing fixups */
-			if (mc->mc_flags & C_SUB) {
-				dummy.mc_flags = C_INITIALIZED;
-				dummy.mc_next = mc->mc_txn->mt_cursors[mc->mc_dbi];
-				mc->mc_txn->mt_cursors[mc->mc_dbi] = &dummy;
-				dummy.mc_xcursor = (MDB_xcursor *)&mn;
-			} else {
-				mn.mc_next = mc->mc_txn->mt_cursors[mc->mc_dbi];
-				mc->mc_txn->mt_cursors[mc->mc_dbi] = &mn;
-			}
+			CURSOR_TMP_TRACK(mc, mn, dummy, tracked);
 			rc = mdb_page_merge(mc, &mn);
-			if (mc->mc_flags & C_SUB)
-				mc->mc_txn->mt_cursors[mc->mc_dbi] = dummy.mc_next;
-			else
-				mc->mc_txn->mt_cursors[mc->mc_dbi] = mn.mc_next;
+			CURSOR_TMP_UNTRACK(mc, tracked);
 			mdb_cursor_copy(&mn, mc);
 		}
 		mc->mc_flags &= ~C_EOF;
@@ -8581,10 +8596,14 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 	 */
 	if (SIZELEFT(mn.mc_pg[ptop]) < mdb_branch_size(env, &sepkey)) {
 		int snum = mc->mc_snum;
+		MDB_cursor dummy, *tracked;
 		mn.mc_snum--;
 		mn.mc_top--;
 		did_split = 1;
+		/* We want other splits to find mn when doing fixups */
+		CURSOR_TMP_TRACK(mc, mn, dummy, tracked);
 		rc = mdb_page_split(&mn, &sepkey, NULL, rp->mp_pgno, 0);
+		CURSOR_TMP_UNTRACK(mc, tracked);
 		if (rc)
 			goto done;
 

From fd598083a77e03b6b98d609d49f7990b16cc81b3 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 23 Nov 2015 01:33:00 +0000
Subject: [PATCH 163/504] ITS#8321 simplify page_split fixups

---
 libraries/liblmdb/mdb.c | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 7ac24610ff..fe97b3ce10 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -8727,7 +8727,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 		/* Adjust other cursors pointing to mp */
 		MDB_cursor *m2, *m3;
 		MDB_dbi dbi = mc->mc_dbi;
-		int fixup = NUMKEYS(mp);
+		nkeys = NUMKEYS(mp);
 
 		for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) {
 			if (mc->mc_flags & C_SUB)
@@ -8740,12 +8740,15 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 				continue;
 			if (new_root) {
 				int k;
+				/* sub cursors may be on different DB */
+				if (m3->mc_pg[0] != mp)
+					continue;
 				/* root split */
 				for (k=new_root; k>=0; k--) {
 					m3->mc_ki[k+1] = m3->mc_ki[k];
 					m3->mc_pg[k+1] = m3->mc_pg[k];
 				}
-				if (m3->mc_ki[0] >= split_indx) {
+				if (m3->mc_ki[0] > nkeys) {
 					m3->mc_ki[0] = 1;
 				} else {
 					m3->mc_ki[0] = 0;
@@ -8757,10 +8760,13 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 			if (m3->mc_top >= mc->mc_top && m3->mc_pg[mc->mc_top] == mp) {
 				if (m3->mc_ki[mc->mc_top] >= newindx && !(nflags & MDB_SPLIT_REPLACE))
 					m3->mc_ki[mc->mc_top]++;
-				if (m3->mc_ki[mc->mc_top] >= fixup) {
+				if (m3->mc_ki[mc->mc_top] >= nkeys) {
 					m3->mc_pg[mc->mc_top] = rp;
-					m3->mc_ki[mc->mc_top] -= fixup;
-					m3->mc_ki[ptop] = mn.mc_ki[ptop];
+					m3->mc_ki[mc->mc_top] -= nkeys;
+					for (i=0; i<mc->mc_top; i++) {
+						m3->mc_ki[i] = mn.mc_ki[i];
+						m3->mc_pg[i] = mn.mc_pg[i];
+					}
 				}
 			} else if (!did_split && m3->mc_top >= ptop && m3->mc_pg[ptop] == mc->mc_pg[ptop] &&
 				m3->mc_ki[ptop] >= mc->mc_ki[ptop]) {

From 6fdaea4165146b9dc8e0365a0477e41b03c4daaf Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 23 Nov 2015 01:34:26 +0000
Subject: [PATCH 164/504] ITS#8321 reorganize page_split fixups

DUPFIXED fixups needed to occur after separator update.
MDB_RESERVE handling moved after split fixup.
---
 libraries/liblmdb/mdb.c | 26 +++++++++++++++++++-------
 1 file changed, 19 insertions(+), 7 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index fe97b3ce10..bdb134a533 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -8501,8 +8501,6 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 				rp->mp_lower += sizeof(indx_t);
 				rp->mp_upper -= ksize - sizeof(indx_t);
 				mc->mc_ki[mc->mc_top] = x;
-				mc->mc_pg[mc->mc_top] = rp;
-				mc->mc_ki[ptop]++;
 			}
 		} else {
 			int psize, nsize, k;
@@ -8703,11 +8701,6 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 		/* reset back to original page */
 		if (newindx < split_indx) {
 			mc->mc_pg[mc->mc_top] = mp;
-			if (nflags & MDB_RESERVE) {
-				node = NODEPTR(mp, mc->mc_ki[mc->mc_top]);
-				if (!(node->mn_flags & F_BIGDATA))
-					newdata->mv_data = NODEDATA(node);
-			}
 		} else {
 			mc->mc_pg[mc->mc_top] = rp;
 			mc->mc_ki[ptop]++;
@@ -8721,6 +8714,25 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 				}
 			}
 		}
+		if (nflags & MDB_RESERVE) {
+			node = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
+			if (!(node->mn_flags & F_BIGDATA))
+				newdata->mv_data = NODEDATA(node);
+		}
+	} else {
+		if (newindx >= split_indx) {
+			mc->mc_pg[mc->mc_top] = rp;
+			mc->mc_ki[ptop]++;
+			/* Make sure mc_ki is still valid.
+			 */
+			if (mn.mc_pg[ptop] != mc->mc_pg[ptop] &&
+				mc->mc_ki[ptop] >= NUMKEYS(mc->mc_pg[ptop])) {
+				for (i=0; i<=ptop; i++) {
+					mc->mc_pg[i] = mn.mc_pg[i];
+					mc->mc_ki[i] = mn.mc_ki[i];
+				}
+			}
+		}
 	}
 
 	{

From e29cfa0b4992025c15eec1af4a59599c040aa2e9 Mon Sep 17 00:00:00 2001
From: Orivej Desh <orivej@gmx.fr>
Date: Sun, 22 Nov 2015 00:59:55 +0000
Subject: [PATCH 165/504] ITS#8319 mdb_load: explain readline and
 mdb_cursor_put errors

---
 libraries/liblmdb/mdb_load.c | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c
index 160bc1b885..5373a9d5d6 100644
--- a/libraries/liblmdb/mdb_load.c
+++ b/libraries/liblmdb/mdb_load.c
@@ -400,20 +400,22 @@ int main(int argc, char *argv[])
 
 		while(1) {
 			rc = readline(&key, &kbuf);
-			if (rc == EOF)
+			if (rc)  /* rc == EOF */
 				break;
-			if (rc)
-				goto txn_abort;
 
 			rc = readline(&data, &dbuf);
-			if (rc)
+			if (rc) {
+				fprintf(stderr, "%s: line %" Z "d: failed to read key value\n", prog, lineno);
 				goto txn_abort;
-			
+			}
+
 			rc = mdb_cursor_put(mc, &key, &data, putflags);
 			if (rc == MDB_KEYEXIST && putflags)
 				continue;
-			if (rc)
+			if (rc) {
+				fprintf(stderr, "mdb_cursor_put failed, error %d %s\n", rc, mdb_strerror(rc));
 				goto txn_abort;
+			}
 			batch++;
 			if (batch == 100) {
 				rc = mdb_txn_commit(txn);

From 85d2f9804ae76a24702b91724ecbd1bf7479e2e3 Mon Sep 17 00:00:00 2001
From: Orivej Desh <orivej@gmx.fr>
Date: Sun, 22 Nov 2015 01:15:14 +0000
Subject: [PATCH 166/504] ITS#8320 mdb_load: fix loading data from simple text
 files

mdb_load -T was supposed to read escaped text, but 21b51cb7 "Add mdb_load"
made it read hex.
---
 libraries/liblmdb/mdb_load.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c
index 5373a9d5d6..d2f0968188 100644
--- a/libraries/liblmdb/mdb_load.c
+++ b/libraries/liblmdb/mdb_load.c
@@ -327,7 +327,7 @@ int main(int argc, char *argv[])
 			putflags = MDB_NOOVERWRITE|MDB_NODUPDATA;
 			break;
 		case 'T':
-			mode |= NOHDR;
+			mode |= NOHDR | PRINT;
 			break;
 		default:
 			usage();

From fae01c8df524b2317954e1b1a2fb14bcdc47e1fa Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Mon, 23 Nov 2015 13:03:09 +0100
Subject: [PATCH 167/504] CURSOR_TMP_[UN]TRACK() -> WITH_CURSOR_TRACKING()

---
 libraries/liblmdb/mdb.c | 46 +++++++++++++++++------------------------
 1 file changed, 19 insertions(+), 27 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index bdb134a533..6c4a4dace3 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7645,21 +7645,21 @@ mdb_update_key(MDB_cursor *mc, MDB_val *key)
 static void
 mdb_cursor_copy(const MDB_cursor *csrc, MDB_cursor *cdst);
 
-/** Track a temporary cursor */
-#define CURSOR_TMP_TRACK(mc, mn, dummy, tracked) \
-	if (mc->mc_flags & C_SUB) { \
+/** Perform \b act while tracking temporary cursor \b mn */
+#define WITH_CURSOR_TRACKING(mn, act) do { \
+	MDB_cursor dummy, *tracked, **tp = &(mn).mc_txn->mt_cursors[mn.mc_dbi]; \
+	if ((mn).mc_flags & C_SUB) { \
 		dummy.mc_flags =  C_INITIALIZED; \
-		dummy.mc_xcursor = (MDB_xcursor *)&mn; \
+		dummy.mc_xcursor = (MDB_xcursor *)&(mn);	\
 		tracked = &dummy; \
 	} else { \
-		tracked = &mn; \
+		tracked = &(mn); \
 	} \
-	tracked->mc_next = mc->mc_txn->mt_cursors[mc->mc_dbi]; \
-	mc->mc_txn->mt_cursors[mc->mc_dbi] = tracked
-
-/** Stop tracking a temporary cursor */
-#define CURSOR_TMP_UNTRACK(mc, tracked) \
-	mc->mc_txn->mt_cursors[mc->mc_dbi] = tracked->mc_next
+	tracked->mc_next = *tp; \
+	*tp = tracked; \
+	{ act; } \
+	*tp = tracked->mc_next; \
+} while (0)
 
 /** Move a node from csrc to cdst.
  */
@@ -7816,7 +7816,6 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft)
 	 */
 	if (csrc->mc_ki[csrc->mc_top] == 0) {
 		if (csrc->mc_ki[csrc->mc_top-1] != 0) {
-			MDB_cursor dummy, *tracked;
 			if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) {
 				key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], 0, key.mv_size);
 			} else {
@@ -7830,9 +7829,8 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft)
 			mn.mc_snum--;
 			mn.mc_top--;
 			/* We want mdb_rebalance to find mn when doing fixups */
-			CURSOR_TMP_TRACK(csrc, mn, dummy, tracked);
-			rc = mdb_update_key(&mn, &key);
-			CURSOR_TMP_UNTRACK(csrc, tracked);
+			WITH_CURSOR_TRACKING(mn,
+				rc = mdb_update_key(&mn, &key));
 			if (rc)
 				return rc;
 		}
@@ -7849,7 +7847,6 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft)
 
 	if (cdst->mc_ki[cdst->mc_top] == 0) {
 		if (cdst->mc_ki[cdst->mc_top-1] != 0) {
-			MDB_cursor dummy, *tracked;
 			if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) {
 				key.mv_data = LEAF2KEY(cdst->mc_pg[cdst->mc_top], 0, key.mv_size);
 			} else {
@@ -7863,9 +7860,8 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft)
 			mn.mc_snum--;
 			mn.mc_top--;
 			/* We want mdb_rebalance to find mn when doing fixups */
-			CURSOR_TMP_TRACK(cdst, mn, dummy, tracked);
-			rc = mdb_update_key(&mn, &key);
-			CURSOR_TMP_UNTRACK(cdst, tracked);
+			WITH_CURSOR_TRACKING(mn,
+				rc = mdb_update_key(&mn, &key));
 			if (rc)
 				return rc;
 		}
@@ -8224,13 +8220,11 @@ mdb_rebalance(MDB_cursor *mc)
 		if (!fromleft) {
 			rc = mdb_page_merge(&mn, mc);
 		} else {
-			MDB_cursor dummy, *tracked;
 			oldki += NUMKEYS(mn.mc_pg[mn.mc_top]);
 			mn.mc_ki[mn.mc_top] += mc->mc_ki[mn.mc_top] + 1;
 			/* We want mdb_rebalance to find mn when doing fixups */
-			CURSOR_TMP_TRACK(mc, mn, dummy, tracked);
-			rc = mdb_page_merge(mc, &mn);
-			CURSOR_TMP_UNTRACK(mc, tracked);
+			WITH_CURSOR_TRACKING(mn,
+				rc = mdb_page_merge(mc, &mn));
 			mdb_cursor_copy(&mn, mc);
 		}
 		mc->mc_flags &= ~C_EOF;
@@ -8594,14 +8588,12 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 	 */
 	if (SIZELEFT(mn.mc_pg[ptop]) < mdb_branch_size(env, &sepkey)) {
 		int snum = mc->mc_snum;
-		MDB_cursor dummy, *tracked;
 		mn.mc_snum--;
 		mn.mc_top--;
 		did_split = 1;
 		/* We want other splits to find mn when doing fixups */
-		CURSOR_TMP_TRACK(mc, mn, dummy, tracked);
-		rc = mdb_page_split(&mn, &sepkey, NULL, rp->mp_pgno, 0);
-		CURSOR_TMP_UNTRACK(mc, tracked);
+		WITH_CURSOR_TRACKING(mn,
+			rc = mdb_page_split(&mn, &sepkey, NULL, rp->mp_pgno, 0));
 		if (rc)
 			goto done;
 

From 83258ca81efbfdbccc2b09c4539b9babe5c27726 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 23 Nov 2015 13:11:29 +0000
Subject: [PATCH 168/504] ITS#8323 Fix nested commit

Must remove our spilled pages from parent's dirty list
---
 libraries/liblmdb/mdb.c | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 6c4a4dace3..fe78eb9fa8 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -3500,6 +3500,25 @@ mdb_txn_commit(MDB_txn *txn)
 			pspill[0] = y;
 		}
 
+		/* Remove anything in our spill list from parent's dirty list */
+		if (txn->mt_spill_pgs && txn->mt_spill_pgs[0]) {
+			for (i=1; i<=txn->mt_spill_pgs[0]; i++) {
+				MDB_ID pn = txn->mt_spill_pgs[i];
+				if (pn & 1)
+					continue;	/* deleted spillpg */
+				pn >>= 1;
+				y = mdb_mid2l_search(dst, pn);
+				if (y <= dst[0].mid && dst[y].mid == pn) {
+					free(dst[y].mptr);
+					while (y < dst[0].mid) {
+						dst[y] = dst[y+1];
+						y++;
+					}
+					dst[0].mid--;
+				}
+			}
+		}
+
 		/* Find len = length of merging our dirty list with parent's */
 		x = dst[0].mid;
 		dst[0].mid = 0;		/* simplify loops */

From b5ce8108e8bb92df00b2d27ef0e37bfe756d3752 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 24 Nov 2015 12:54:46 +0000
Subject: [PATCH 169/504] ITS#8321 deinit empty cursors

Always unset C_INIT flag if the cursor's target DB has been deleted
---
 libraries/liblmdb/mdb.c | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index fe78eb9fa8..43bc85df1c 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5284,8 +5284,11 @@ mdb_cursor_pop(MDB_cursor *mc)
 			mc->mc_pg[mc->mc_top]->mp_pgno, DDBI(mc), (void *) mc));
 
 		mc->mc_snum--;
-		if (mc->mc_snum)
+		if (mc->mc_snum) {
 			mc->mc_top--;
+		} else {
+			mc->mc_flags &= ~C_INITIALIZED;
+		}
 	}
 }
 
@@ -6941,6 +6944,7 @@ mdb_cursor_del(MDB_cursor *mc, unsigned int flags)
 		if (flags & MDB_NODUPDATA) {
 			/* mdb_cursor_del0() will subtract the final entry */
 			mc->mc_db->md_entries -= mc->mc_xcursor->mx_db.md_entries - 1;
+			mc->mc_xcursor->mx_cursor.mc_flags &= ~C_INITIALIZED;
 		} else {
 			if (!F_ISSET(leaf->mn_flags, F_SUBDATA)) {
 				mc->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf);
@@ -6978,6 +6982,8 @@ mdb_cursor_del(MDB_cursor *mc, unsigned int flags)
 				mc->mc_db->md_entries--;
 				mc->mc_flags |= C_DEL;
 				return rc;
+			} else {
+				mc->mc_xcursor->mx_cursor.mc_flags &= ~C_INITIALIZED;
 			}
 			/* otherwise fall thru and delete the sub-DB */
 		}
@@ -9767,6 +9773,7 @@ done:
 	} else if (rc == MDB_NOTFOUND) {
 		rc = MDB_SUCCESS;
 	}
+	mc->mc_flags &= ~C_INITIALIZED;
 	return rc;
 }
 

From cf8677e1b827c3ba1303433c041840886a9b146e Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 23 Nov 2015 18:30:24 +0000
Subject: [PATCH 170/504] ITS#8321 Fix mdb_cursor_set

Always reinit mc_pg[0] if cursor is not C_INITIALIZED
It might have a stale value when using nested txns
---
 libraries/liblmdb/mdb.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 43bc85df1c..5cddc3d47c 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5999,6 +5999,8 @@ mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data,
 			} else
 				return MDB_NOTFOUND;
 		}
+	} else {
+		mc->mc_pg[0] = 0;
 	}
 
 	rc = mdb_page_search(mc, key, 0);

From db02a71dd80291fd836584b540075acad457cb74 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 24 Nov 2015 01:21:05 +0000
Subject: [PATCH 171/504] ITS#8321 mdb_put cursor needs tracking too

---
 libraries/liblmdb/mdb.c | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 5cddc3d47c..4cc78ba003 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -8779,7 +8779,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 					m3->mc_ki[k+1] = m3->mc_ki[k];
 					m3->mc_pg[k+1] = m3->mc_pg[k];
 				}
-				if (m3->mc_ki[0] > nkeys) {
+				if (m3->mc_ki[0] >= nkeys) {
 					m3->mc_ki[0] = 1;
 				} else {
 					m3->mc_ki[0] = 0;
@@ -8821,6 +8821,7 @@ mdb_put(MDB_txn *txn, MDB_dbi dbi,
 {
 	MDB_cursor mc;
 	MDB_xcursor mx;
+	int rc;
 
 	if (!key || !data || !TXN_DBI_EXIST(txn, dbi, DB_USRVALID))
 		return EINVAL;
@@ -8832,7 +8833,11 @@ mdb_put(MDB_txn *txn, MDB_dbi dbi,
 		return (txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN;
 
 	mdb_cursor_init(&mc, txn, dbi, &mx);
-	return mdb_cursor_put(&mc, key, data, flags);
+	mc.mc_next = txn->mt_cursors[dbi];
+	txn->mt_cursors[dbi] = &mc;
+	rc = mdb_cursor_put(&mc, key, data, flags);
+	txn->mt_cursors[dbi] = mc.mc_next;
+	return rc;
 }
 
 #ifndef MDB_WBUF

From 69829288dc2c59287de290f6ab57b0c5d10a212b Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 23 Nov 2015 13:19:26 +0000
Subject: [PATCH 172/504] ITS#8321 page_touch - don't fixup the cursor we just
 touched

---
 libraries/liblmdb/mdb.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 4cc78ba003..4eae413282 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2487,6 +2487,7 @@ done:
 	} else {
 		for (; m2; m2=m2->mc_next) {
 			if (m2->mc_snum < mc->mc_snum) continue;
+			if (m2 == mc) continue;
 			if (m2->mc_pg[mc->mc_top] == mp) {
 				m2->mc_pg[mc->mc_top] = np;
 				if ((mc->mc_db->md_flags & MDB_DUPSORT) &&

From 1424aa06a6fbd580b0ac92cd8ec3a81b5a1d0e12 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 23 Nov 2015 16:58:57 +0000
Subject: [PATCH 173/504] ITS#8321 More cursor fixup

Based on page_touch fixup from ITS#7594 but expanded:
make sure sub-cursors agree with main cursors.
---
 libraries/liblmdb/mdb.c | 45 ++++++++++++++++++++++++++++++++++++++---
 1 file changed, 42 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 4eae413282..0a5608cb5d 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1682,6 +1682,13 @@ mdb_cursor_chk(MDB_cursor *mc)
 	}
 	if (mc->mc_ki[i] >= NUMKEYS(mc->mc_pg[i]))
 		printf("ack!\n");
+	if (mc->mc_xcursor && (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) {
+		node = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
+		if (((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) &&
+			mc->mc_xcursor->mx_cursor.mc_pg[0] != NODEDATA(node)) {
+			printf("blah!\n");
+		}
+	}
 }
 #endif
 
@@ -2492,10 +2499,10 @@ done:
 				m2->mc_pg[mc->mc_top] = np;
 				if ((mc->mc_db->md_flags & MDB_DUPSORT) &&
 					IS_LEAF(np) &&
-					m2->mc_ki[mc->mc_top] == mc->mc_ki[mc->mc_top])
+					(m2->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED))
 				{
-					MDB_node *leaf = NODEPTR(np, mc->mc_ki[mc->mc_top]);
-					if (!(leaf->mn_flags & F_SUBDATA))
+					MDB_node *leaf = NODEPTR(np, m2->mc_ki[mc->mc_top]);
+					if ((leaf->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA)
 						m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf);
 				}
 			}
@@ -7744,6 +7751,7 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft)
 		data.mv_size = NODEDSZ(srcnode);
 		data.mv_data = NODEDATA(srcnode);
 	}
+	mn.mc_xcursor = NULL;
 	if (IS_BRANCH(cdst->mc_pg[cdst->mc_top]) && cdst->mc_ki[cdst->mc_top] == 0) {
 		unsigned int snum = cdst->mc_snum;
 		MDB_node *s2;
@@ -7815,6 +7823,12 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft)
 					m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top];
 					m3->mc_ki[csrc->mc_top-1]++;
 				}
+				if (m3->mc_xcursor && (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) &&
+					IS_LEAF(mps)) {
+					MDB_node *node = NODEPTR(m3->mc_pg[csrc->mc_top], m3->mc_ki[csrc->mc_top]);
+					if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA)
+						m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node);
+				}
 			}
 		} else
 		/* Adding on the right, bump others down */
@@ -7835,6 +7849,12 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft)
 					} else {
 						m3->mc_ki[csrc->mc_top]--;
 					}
+					if (m3->mc_xcursor && (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) &&
+						IS_LEAF(mps)) {
+						MDB_node *node = NODEPTR(m3->mc_pg[csrc->mc_top], m3->mc_ki[csrc->mc_top]);
+						if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA)
+							m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node);
+					}
 				}
 			}
 		}
@@ -7959,6 +7979,7 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst)
 				MDB_cursor mn;
 				MDB_node *s2;
 				mdb_cursor_copy(csrc, &mn);
+				mn.mc_xcursor = NULL;
 				/* must find the lowest key below src */
 				rc = mdb_page_search_lowest(&mn);
 				if (rc)
@@ -8034,6 +8055,12 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst)
 				m3->mc_ki[top-1] > csrc->mc_ki[top-1]) {
 				m3->mc_ki[top-1]--;
 			}
+			if (m3->mc_xcursor && (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) &&
+				IS_LEAF(psrc)) {
+				MDB_node *node = NODEPTR(m3->mc_pg[top], m3->mc_ki[top]);
+				if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA)
+					m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node);
+			}
 		}
 	}
 	{
@@ -8292,6 +8319,11 @@ mdb_cursor_del0(MDB_cursor *mc)
 					else if (mc->mc_db->md_flags & MDB_DUPSORT)
 						m3->mc_xcursor->mx_cursor.mc_flags &= ~C_INITIALIZED;
 				}
+				if (m3->mc_xcursor && (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) {
+					MDB_node *node = NODEPTR(m3->mc_pg[mc->mc_top], m3->mc_ki[mc->mc_top]);
+					if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA)
+						m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node);
+				}
 			}
 		}
 	}
@@ -8474,6 +8506,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 	}
 
 	mdb_cursor_copy(mc, &mn);
+	mn.mc_xcursor = NULL;
 	mn.mc_pg[mn.mc_top] = rp;
 	mn.mc_ki[ptop] = mc->mc_ki[ptop]+1;
 
@@ -8804,6 +8837,12 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 				m3->mc_ki[ptop] >= mc->mc_ki[ptop]) {
 				m3->mc_ki[ptop]++;
 			}
+			if (m3->mc_xcursor && (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) &&
+				IS_LEAF(mp)) {
+				MDB_node *node = NODEPTR(m3->mc_pg[mc->mc_top], m3->mc_ki[mc->mc_top]);
+				if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA)
+					m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node);
+			}
 		}
 	}
 	DPRINTF(("mp left: %d, rp left: %d", SIZELEFT(mp), SIZELEFT(rp)));

From 50d96511f87c8e64f69cfd24d7a2b94e746e442f Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 24 Nov 2015 12:14:49 +0000
Subject: [PATCH 174/504] Cleanup C_DEL flag usage

Only set it if the cursor's current position was deleted
---
 libraries/liblmdb/mdb.c | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 0a5608cb5d..f4268db0f8 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -6226,8 +6226,6 @@ mdb_cursor_get(MDB_cursor *mc, MDB_val *key, MDB_val *data,
 				MDB_GET_KEY(leaf, key);
 				if (data) {
 					if (F_ISSET(leaf->mn_flags, F_DUPDATA)) {
-						if (mc->mc_flags & C_DEL)
-							mdb_xcursor_init1(mc, leaf);
 						rc = mdb_cursor_get(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_GET_CURRENT);
 					} else {
 						rc = mdb_node_read(mc->mc_txn, leaf, data);
@@ -6990,7 +6988,6 @@ mdb_cursor_del(MDB_cursor *mc, unsigned int flags)
 					}
 				}
 				mc->mc_db->md_entries--;
-				mc->mc_flags |= C_DEL;
 				return rc;
 			} else {
 				mc->mc_xcursor->mx_cursor.mc_flags &= ~C_INITIALIZED;
@@ -8312,12 +8309,12 @@ mdb_cursor_del0(MDB_cursor *mc)
 			if (m3 == mc || m3->mc_snum < mc->mc_snum)
 				continue;
 			if (m3->mc_pg[mc->mc_top] == mp) {
-				if (m3->mc_ki[mc->mc_top] >= ki) {
+				if (m3->mc_ki[mc->mc_top] == ki) {
 					m3->mc_flags |= C_DEL;
-					if (m3->mc_ki[mc->mc_top] > ki)
-						m3->mc_ki[mc->mc_top]--;
-					else if (mc->mc_db->md_flags & MDB_DUPSORT)
+					if (mc->mc_db->md_flags & MDB_DUPSORT)
 						m3->mc_xcursor->mx_cursor.mc_flags &= ~C_INITIALIZED;
+				} else if (m3->mc_ki[mc->mc_top] > ki) {
+					m3->mc_ki[mc->mc_top]--;
 				}
 				if (m3->mc_xcursor && (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) {
 					MDB_node *node = NODEPTR(m3->mc_pg[mc->mc_top], m3->mc_ki[mc->mc_top]);

From da67af1a4bf1a6d3f5d173d9daf74bb4ec66175a Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 24 Nov 2015 15:09:49 +0000
Subject: [PATCH 175/504] ITS#8321 fix ambiguity in cursor_put fixup

After delete/add of a node, other nodes may no longer be
pointing at the data they intended. This can confuse subsequent
fixups.
---
 libraries/liblmdb/mdb.c | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index f4268db0f8..4844b0d7f9 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -6804,22 +6804,28 @@ new_sub:
 	} else {
 		/* There is room already in this leaf page. */
 		rc = mdb_node_add(mc, mc->mc_ki[mc->mc_top], key, rdata, 0, nflags);
-		if (rc == 0 && insert_key) {
+		if (rc == 0) {
 			/* Adjust other cursors pointing to mp */
 			MDB_cursor *m2, *m3;
 			MDB_dbi dbi = mc->mc_dbi;
 			unsigned i = mc->mc_top;
 			MDB_page *mp = mc->mc_pg[i];
+			int nkeys = NUMKEYS(mp);
 
 			for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) {
 				if (mc->mc_flags & C_SUB)
 					m3 = &m2->mc_xcursor->mx_cursor;
 				else
 					m3 = m2;
-				if (m3 == mc || m3->mc_snum < mc->mc_snum) continue;
-				if (m3->mc_pg[i] == mp && m3->mc_ki[i] >= mc->mc_ki[i]) {
+				if (m3 == mc || m3->mc_snum < mc->mc_snum || m3->mc_pg[i] != mp) continue;
+				if (m3->mc_ki[i] >= mc->mc_ki[i] && insert_key) {
 					m3->mc_ki[i]++;
 				}
+				if (m3->mc_xcursor && (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) {
+					MDB_node *n2 = NODEPTR(mp, m3->mc_ki[i]);
+					if ((n2->mn_flags & (F_SUBDATA|F_DUPDATA)) == F_DUPDATA)
+						m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(n2);
+				}
 			}
 		}
 	}

From 3a4fd79cbfe73d049da7e52d36a12ab5d850b5b9 Mon Sep 17 00:00:00 2001
From: Heiko Becker <heirecka@exherbo.org>
Date: Thu, 11 Jun 2015 20:44:06 +0200
Subject: [PATCH 176/504] ITS#8168 Allow passing AR to make

This is helpful when the ar executable is named differently, for
example with an arch specific prefix.
---
 libraries/liblmdb/Makefile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile
index 2c3817f049..e3daf9513f 100644
--- a/libraries/liblmdb/Makefile
+++ b/libraries/liblmdb/Makefile
@@ -19,6 +19,7 @@
 # read mdb.c before changing any of them.
 #
 CC	= gcc
+AR	= ar
 W	= -W -Wall -Wno-unused-parameter -Wbad-function-cast -Wuninitialized
 THREADS = -pthread
 OPT = -O2 -g
@@ -54,7 +55,7 @@ test:	all
 	./mtest && ./mdb_stat testdb
 
 liblmdb.a:	mdb.o midl.o
-	ar rs $@ mdb.o midl.o
+	$(AR) rs $@ mdb.o midl.o
 
 liblmdb.so:	mdb.lo midl.lo
 #	$(CC) $(LDFLAGS) -pthread -shared -Wl,-Bsymbolic -o $@ mdb.o midl.o $(SOLIBS)

From 6cff716f6024f98ecd743feb50c406a5136ed741 Mon Sep 17 00:00:00 2001
From: Heiko Becker <heirecka@exherbo.org>
Date: Thu, 11 Jun 2015 21:09:59 +0200
Subject: [PATCH 177/504] ITS#8169 Allow passing mandir to make install

The motivation for this change is my distribution moving to a multiarch
layout. While the architecture specific stuff (binaries, libraries,
etc.) is installed under /usr/${host}/{bin,lib,...} architecture-independent
data should still be installed to /usr/share/.
---
 libraries/liblmdb/Makefile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile
index e3daf9513f..098bff5d86 100644
--- a/libraries/liblmdb/Makefile
+++ b/libraries/liblmdb/Makefile
@@ -27,6 +27,7 @@ CFLAGS	= $(THREADS) $(OPT) $(W) $(XCFLAGS)
 LDLIBS	=
 SOLIBS	=
 prefix	= /usr/local
+mandir = $(prefix)/man
 
 ########################################################################
 
@@ -45,7 +46,7 @@ install: $(ILIBS) $(IPROGS) $(IHDRS)
 	for f in $(IPROGS); do cp $$f $(DESTDIR)$(prefix)/bin; done
 	for f in $(ILIBS); do cp $$f $(DESTDIR)$(prefix)/lib; done
 	for f in $(IHDRS); do cp $$f $(DESTDIR)$(prefix)/include; done
-	for f in $(IDOCS); do cp $$f $(DESTDIR)$(prefix)/man/man1; done
+	for f in $(IDOCS); do cp $$f $(DESTDIR)$(mandir)/man1; done
 
 clean:
 	rm -rf $(PROGS) *.[ao] *.[ls]o *~ testdb

From dbbbfa10be002b8d0bbf04c08f6dbfb359fc00eb Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 24 Nov 2015 20:21:51 +0000
Subject: [PATCH 178/504] ITS#8321 cleanup unused var

from da67af1a4bf1a6d3f5d173d9daf74bb4ec66175a
---
 libraries/liblmdb/mdb.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 4844b0d7f9..f712115709 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -6810,7 +6810,6 @@ new_sub:
 			MDB_dbi dbi = mc->mc_dbi;
 			unsigned i = mc->mc_top;
 			MDB_page *mp = mc->mc_pg[i];
-			int nkeys = NUMKEYS(mp);
 
 			for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) {
 				if (mc->mc_flags & C_SUB)

From fb5a768a77ca5330e15a3a34ceb694bc11cb216a Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 30 Nov 2015 18:46:19 +0000
Subject: [PATCH 179/504] ITS#8324 incremental DB file growth for Windows

---
 libraries/liblmdb/Makefile |   4 +-
 libraries/liblmdb/mdb.c    | 101 +++++++++++++++++++++++++------------
 2 files changed, 72 insertions(+), 33 deletions(-)

diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile
index 098bff5d86..0d343674e9 100644
--- a/libraries/liblmdb/Makefile
+++ b/libraries/liblmdb/Makefile
@@ -24,8 +24,8 @@ W	= -W -Wall -Wno-unused-parameter -Wbad-function-cast -Wuninitialized
 THREADS = -pthread
 OPT = -O2 -g
 CFLAGS	= $(THREADS) $(OPT) $(W) $(XCFLAGS)
-LDLIBS	=
-SOLIBS	=
+LDLIBS	= # -lntdll # Windows needs ntdll
+SOLIBS	= # -lntdll
 prefix	= /usr/local
 mandir = $(prefix)/man
 
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index f712115709..2e940a156a 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -38,6 +38,36 @@
 #ifdef _WIN32
 #include <malloc.h>
 #include <windows.h>
+
+/* We use native NT APIs to setup the memory map, so that we can
+ * let the DB file grow incrementally instead of always preallocating
+ * the full size. These APIs are defined in <wdm.h> and <ntifs.h>
+ * but those headers are meant for driver-level development and
+ * conflict with the regular user-level headers, so we explicitly
+ * declare them here. Using these APIs also means we must link to
+ * ntdll.dll, which is not linked by default in user code.
+ */
+NTSTATUS WINAPI
+NtCreateSection(OUT PHANDLE sh, IN ACCESS_MASK acc,
+  IN void * oa OPTIONAL,
+  IN PLARGE_INTEGER ms OPTIONAL,
+  IN ULONG pp, IN ULONG aa, IN HANDLE fh OPTIONAL);
+
+typedef enum _SECTION_INHERIT {
+	ViewShare = 1,
+	ViewUnmap = 2
+} SECTION_INHERIT;
+
+NTSTATUS WINAPI
+NtMapViewOfSection(IN PHANDLE sh, IN HANDLE ph,
+  IN OUT PVOID *addr, IN ULONG_PTR zbits,
+  IN SIZE_T cs, IN OUT PLARGE_INTEGER off OPTIONAL,
+  IN OUT PSIZE_T vs, IN SECTION_INHERIT ih,
+  IN ULONG at, IN ULONG pp);
+
+NTSTATUS WINAPI
+NtClose(HANDLE h);
+
 /** getpid() returns int; MinGW defines pid_t but MinGW64 typedefs it
  *  as int64 which is wrong. MSVC doesn't define it at all, so just
  *  don't use it.
@@ -2293,6 +2323,20 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
 			rc = MDB_MAP_FULL;
 			goto fail;
 	}
+#ifdef _WIN32
+	{
+		void *p;
+		p = (MDB_page *)(env->me_map + env->me_psize * pgno);
+		p = VirtualAlloc(p, env->me_psize * num, MEM_COMMIT,
+			(env->me_flags & MDB_WRITEMAP) ? PAGE_READWRITE:
+			PAGE_READONLY);
+		if (!p) {
+			DPUTS("VirtualAlloc failed");
+			rc = ErrCode();
+			goto fail;
+		}
+	}
+#endif
 
 search_done:
 	if (env->me_flags & MDB_WRITEMAP) {
@@ -3956,42 +4000,26 @@ mdb_env_map(MDB_env *env, void *addr)
 	unsigned int flags = env->me_flags;
 #ifdef _WIN32
 	int rc;
+	int access = SECTION_MAP_READ;
 	HANDLE mh;
-	LONG sizelo, sizehi;
-	size_t msize;
-
-	if (flags & MDB_RDONLY) {
-		/* Don't set explicit map size, use whatever exists */
-		msize = 0;
-		sizelo = 0;
-		sizehi = 0;
-	} else {
-		msize = env->me_mapsize;
-		sizelo = msize & 0xffffffff;
-		sizehi = msize >> 16 >> 16; /* only needed on Win64 */
-
-		/* Windows won't create mappings for zero length files.
-		 * and won't map more than the file size.
-		 * Just set the maxsize right now.
-		 */
-		if (SetFilePointer(env->me_fd, sizelo, &sizehi, 0) != (DWORD)sizelo
-			|| !SetEndOfFile(env->me_fd)
-			|| SetFilePointer(env->me_fd, 0, NULL, 0) != 0)
-			return ErrCode();
+	void *map;
+	size_t msize = 0;
+	ULONG pageprot = PAGE_READONLY;
+	if (flags & MDB_WRITEMAP) {
+		access |= SECTION_MAP_WRITE;
+		pageprot = PAGE_READWRITE;
 	}
 
-	mh = CreateFileMapping(env->me_fd, NULL, flags & MDB_WRITEMAP ?
-		PAGE_READWRITE : PAGE_READONLY,
-		sizehi, sizelo, NULL);
-	if (!mh)
-		return ErrCode();
-	env->me_map = MapViewOfFileEx(mh, flags & MDB_WRITEMAP ?
-		FILE_MAP_WRITE : FILE_MAP_READ,
-		0, 0, msize, addr);
-	rc = env->me_map ? 0 : ErrCode();
-	CloseHandle(mh);
+	rc = NtCreateSection(&mh, access, NULL, NULL, PAGE_READWRITE, SEC_RESERVE, env->me_fd);
 	if (rc)
 		return rc;
+	map = addr;
+	msize = env->me_mapsize;
+	rc = NtMapViewOfSection(mh, GetCurrentProcess(), &map, 0, 0, NULL, &msize, ViewUnmap, MEM_RESERVE, pageprot);
+	NtClose(mh);
+	if (rc)
+		return rc;
+	env->me_map = map;
 #else
 	int prot = PROT_READ;
 	if (flags & MDB_WRITEMAP) {
@@ -4228,6 +4256,17 @@ mdb_env_open2(MDB_env *env)
 			return rc;
 		newenv = 0;
 	}
+#ifdef _WIN32
+	/* For FIXEDMAP, make sure the file is non-empty before we attempt to map it */
+	if (newenv) {
+		char dummy = 0;
+		rc = WriteFile(env->me_fd, &dummy, 1, NULL, NULL);
+		if (!rc) {
+			rc = ErrCode();
+			return rc;
+		}
+	}
+#endif
 
 	rc = mdb_env_map(env, (flags & MDB_FIXEDMAP) ? meta.mm_address : NULL);
 	if (rc)

From 50949b460625e1b0768d8bccb89303999d45a881 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 1 Dec 2015 13:48:29 +0000
Subject: [PATCH 180/504] ITS#8169 more for prev patch

Missed the mkdir command
---
 libraries/liblmdb/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile
index 0d343674e9..7bc41ae497 100644
--- a/libraries/liblmdb/Makefile
+++ b/libraries/liblmdb/Makefile
@@ -42,7 +42,7 @@ install: $(ILIBS) $(IPROGS) $(IHDRS)
 	mkdir -p $(DESTDIR)$(prefix)/bin
 	mkdir -p $(DESTDIR)$(prefix)/lib
 	mkdir -p $(DESTDIR)$(prefix)/include
-	mkdir -p $(DESTDIR)$(prefix)/man/man1
+	mkdir -p $(DESTDIR)$(mandir)/man1
 	for f in $(IPROGS); do cp $$f $(DESTDIR)$(prefix)/bin; done
 	for f in $(ILIBS); do cp $$f $(DESTDIR)$(prefix)/lib; done
 	for f in $(IHDRS); do cp $$f $(DESTDIR)$(prefix)/include; done

From 45fe870c2ec421286bcac86b514c326ce10009a2 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 1 Dec 2015 14:10:12 +0000
Subject: [PATCH 181/504] ITS#8169 more Makefile tweaks

Use all the same vars as main OpenLDAP makefiles
---
 libraries/liblmdb/Makefile | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile
index 7bc41ae497..f3c93a2ff5 100644
--- a/libraries/liblmdb/Makefile
+++ b/libraries/liblmdb/Makefile
@@ -27,7 +27,12 @@ CFLAGS	= $(THREADS) $(OPT) $(W) $(XCFLAGS)
 LDLIBS	= # -lntdll # Windows needs ntdll
 SOLIBS	= # -lntdll
 prefix	= /usr/local
-mandir = $(prefix)/man
+exec_prefix = $(prefix)
+bindir = $(exec_prefix)/bin
+libdir = $(exec_prefix)/lib
+includedir = $(prefix)/include
+datarootdir = $(prefix)/share
+mandir = $(datarootdir)/man
 
 ########################################################################
 
@@ -39,13 +44,13 @@ PROGS	= $(IPROGS) mtest mtest2 mtest3 mtest4 mtest5
 all:	$(ILIBS) $(PROGS)
 
 install: $(ILIBS) $(IPROGS) $(IHDRS)
-	mkdir -p $(DESTDIR)$(prefix)/bin
-	mkdir -p $(DESTDIR)$(prefix)/lib
-	mkdir -p $(DESTDIR)$(prefix)/include
+	mkdir -p $(DESTDIR)$(bindir)
+	mkdir -p $(DESTDIR)$(libdir)
+	mkdir -p $(DESTDIR)$(includedir)
 	mkdir -p $(DESTDIR)$(mandir)/man1
-	for f in $(IPROGS); do cp $$f $(DESTDIR)$(prefix)/bin; done
-	for f in $(ILIBS); do cp $$f $(DESTDIR)$(prefix)/lib; done
-	for f in $(IHDRS); do cp $$f $(DESTDIR)$(prefix)/include; done
+	for f in $(IPROGS); do cp $$f $(DESTDIR)$(bindir); done
+	for f in $(ILIBS); do cp $$f $(DESTDIR)$(libdir); done
+	for f in $(IHDRS); do cp $$f $(DESTDIR)$(includedir); done
 	for f in $(IDOCS); do cp $$f $(DESTDIR)$(mandir)/man1; done
 
 clean:

From 4198bbde175c65a7af0eafc46da22fc1c96ceeb6 Mon Sep 17 00:00:00 2001
From: Sebastien Launay <sebastien@slaunay.fr>
Date: Thu, 3 Dec 2015 10:42:36 -0800
Subject: [PATCH 182/504] ITS#8330 Fix robust mutex detection for glibc 2.10
 and 2.11

pthread_mutexattr_setrobust and pthread_mutex_consistent are provided since 2.12 not 2.10:
https://sourceware.org/git/?p=glibc.git;a=commit;h=402cd98775db1478f64c9b0dbe00664b89eb2773
https://sourceware.org/git/?p=glibc.git;a=commit;h=78ee21859939ff75ccf8bbe00499b0c462df2e2d
---
 libraries/liblmdb/mdb.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 2e940a156a..6c88c961b8 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -300,8 +300,8 @@ union semun {
 #  define MDB_USE_ROBUST	0
 # else
 #  define MDB_USE_ROBUST	1
-/* glibc < 2.10 only provided _np API */
-#  if defined(__GLIBC__) && GLIBC_VER < 0x02000a
+/* glibc < 2.12 only provided _np API */
+#  if defined(__GLIBC__) && GLIBC_VER < 0x02000c
 #   define PTHREAD_MUTEX_ROBUST	PTHREAD_MUTEX_ROBUST_NP
 #   define pthread_mutexattr_setrobust(attr, flag)	pthread_mutexattr_setrobust_np(attr, flag)
 #   define pthread_mutex_consistent(mutex)	pthread_mutex_consistent_np(mutex)

From c4e31434c7773ee54f2ffdeb545e5740f56492a1 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@symas.com>
Date: Sun, 6 Dec 2015 20:54:23 +0000
Subject: [PATCH 183/504] ITS#8324 additional tweaks

VirtualAlloc is only needed with WRITEMAP. Regular writes
already extend the mmap implictly.
---
 libraries/liblmdb/mdb.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 6c88c961b8..09f15099d2 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2324,7 +2324,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
 			goto fail;
 	}
 #ifdef _WIN32
-	{
+	if (env->me_flags & MDB_WRITEMAP) {
 		void *p;
 		p = (MDB_page *)(env->me_map + env->me_psize * pgno);
 		p = VirtualAlloc(p, env->me_psize * num, MEM_COMMIT,
@@ -4260,7 +4260,8 @@ mdb_env_open2(MDB_env *env)
 	/* For FIXEDMAP, make sure the file is non-empty before we attempt to map it */
 	if (newenv) {
 		char dummy = 0;
-		rc = WriteFile(env->me_fd, &dummy, 1, NULL, NULL);
+		DWORD len;
+		rc = WriteFile(env->me_fd, &dummy, 1, &len, NULL);
 		if (!rc) {
 			rc = ErrCode();
 			return rc;

From abb13ba176afd8c5da573a8b4b7808082e25b9b8 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 8 Dec 2015 16:43:55 +0100
Subject: [PATCH 184/504] ITS#8334 Fix MDB_APPENDDUP vs. rewrite(single item)

---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 09f15099d2..0c93c371b4 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -6642,7 +6642,7 @@ more:
 #endif
 				/* does data match? */
 				if (!dcmp(data, &olddata)) {
-					if (flags & MDB_NODUPDATA)
+					if (flags & (MDB_NODUPDATA|MDB_APPENDDUP))
 						return MDB_KEYEXIST;
 					/* overwrite it */
 					goto current;

From 58d1fd4c73c96ef3097816e975b3d421ead4d86e Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 8 Dec 2015 18:17:24 +0000
Subject: [PATCH 185/504] ITS#8336 fix page_search_root assert on FreeDB

Let "illegal" branch pages thru on the FreeDB - the condition
is only temporary and will be fixed by the time rebalance finishes.
---
 libraries/liblmdb/mdb.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 0c93c371b4..c603654bcd 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5434,7 +5434,11 @@ mdb_page_search_root(MDB_cursor *mc, MDB_val *key, int flags)
 		indx_t		i;
 
 		DPRINTF(("branch page %"Z"u has %u keys", mp->mp_pgno, NUMKEYS(mp)));
-		mdb_cassert(mc, NUMKEYS(mp) > 1);
+		/* Don't assert on branch pages in the FreeDB. We can get here
+		 * while in the process of rebalancing a FreeDB branch page; we must
+		 * let that proceed. ITS#8336
+		 */
+		mdb_cassert(mc, !mc->mc_dbi || NUMKEYS(mp) > 1);
 		DPRINTF(("found index 0 to page %"Z"u", NODEPGNO(NODEPTR(mp, 0))));
 
 		if (flags & (MDB_PS_FIRST|MDB_PS_LAST)) {

From 8b95e7d13e1cc354888a68dc520fd428b859679c Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 8 Dec 2015 19:35:59 +0000
Subject: [PATCH 186/504] ITS#8324 fix for read-only envs

---
 libraries/liblmdb/mdb.c | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index c603654bcd..728da9b709 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -4003,19 +4003,28 @@ mdb_env_map(MDB_env *env, void *addr)
 	int access = SECTION_MAP_READ;
 	HANDLE mh;
 	void *map;
-	size_t msize = 0;
-	ULONG pageprot = PAGE_READONLY;
+	size_t msize;
+	ULONG pageprot = PAGE_READONLY, secprot, alloctype;
+
 	if (flags & MDB_WRITEMAP) {
 		access |= SECTION_MAP_WRITE;
 		pageprot = PAGE_READWRITE;
 	}
+	if (flags & MDB_RDONLY) {
+		secprot = PAGE_READONLY;
+		msize = 0;
+		alloctype = 0;
+	} else {
+		secprot = PAGE_READWRITE;
+		msize = env->me_mapsize;
+		alloctype = MEM_RESERVE;
+	}
 
-	rc = NtCreateSection(&mh, access, NULL, NULL, PAGE_READWRITE, SEC_RESERVE, env->me_fd);
+	rc = NtCreateSection(&mh, access, NULL, NULL, secprot, SEC_RESERVE, env->me_fd);
 	if (rc)
 		return rc;
 	map = addr;
-	msize = env->me_mapsize;
-	rc = NtMapViewOfSection(mh, GetCurrentProcess(), &map, 0, 0, NULL, &msize, ViewUnmap, MEM_RESERVE, pageprot);
+	rc = NtMapViewOfSection(mh, GetCurrentProcess(), &map, 0, 0, NULL, &msize, ViewUnmap, alloctype, pageprot);
 	NtClose(mh);
 	if (rc)
 		return rc;

From 791badd09680cd9cd4c38b13b2dd829fff0ad32d Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 12 Dec 2015 19:25:06 +0100
Subject: [PATCH 187/504] mdb_dbi_open(): Catch strdup failure

---
 libraries/liblmdb/mdb.c | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 728da9b709..67a05c2e66 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -9650,6 +9650,7 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db
 	MDB_db dummy;
 	int rc, dbflag, exact;
 	unsigned int unused = 0, seq;
+	char *namedup;
 	size_t len;
 
 	if (flags & ~VALID_FLAGS)
@@ -9711,8 +9712,16 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db
 		MDB_node *node = NODEPTR(mc.mc_pg[mc.mc_top], mc.mc_ki[mc.mc_top]);
 		if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) != F_SUBDATA)
 			return MDB_INCOMPATIBLE;
-	} else if (rc == MDB_NOTFOUND && (flags & MDB_CREATE)) {
-		/* Create if requested */
+	} else if (! (rc == MDB_NOTFOUND && (flags & MDB_CREATE))) {
+		return rc;
+	}
+
+	/* Done here so we cannot fail after creating a new DB */
+	if ((namedup = strdup(name)) == NULL)
+		return ENOMEM;
+
+	if (rc) {
+		/* MDB_NOTFOUND and MDB_CREATE: Create new DB */
 		data.mv_size = sizeof(MDB_db);
 		data.mv_data = &dummy;
 		memset(&dummy, 0, sizeof(dummy));
@@ -9722,10 +9731,12 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db
 		dbflag |= DB_DIRTY;
 	}
 
-	/* OK, got info, add to table */
-	if (rc == MDB_SUCCESS) {
+	if (rc) {
+		free(namedup);
+	} else {
+		/* Got info, register DBI in this txn */
 		unsigned int slot = unused ? unused : txn->mt_numdbs;
-		txn->mt_dbxs[slot].md_name.mv_data = strdup(name);
+		txn->mt_dbxs[slot].md_name.mv_data = namedup;
 		txn->mt_dbxs[slot].md_name.mv_size = len;
 		txn->mt_dbxs[slot].md_rel = NULL;
 		txn->mt_dbflags[slot] = dbflag;

From c7a786eb00a8af61b16697be17d22eae1e37d102 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 15 Dec 2015 18:45:34 +0000
Subject: [PATCH 188/504] ITS#7992 cleanup

check for utf8_to_utf16 failures
---
 libraries/liblmdb/mdb.c | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 67a05c2e66..57c6a989c6 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -4597,7 +4597,9 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 
 #ifdef _WIN32
 	wchar_t *wlpath;
-	utf8_to_utf16(lpath, -1, &wlpath, NULL);
+	rc = utf8_to_utf16(lpath, -1, &wlpath, NULL);
+	if (rc)
+		return rc;
 	env->me_lfd = CreateFileW(wlpath, GENERIC_READ|GENERIC_WRITE,
 		FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, OPEN_ALWAYS,
 		FILE_ATTRIBUTE_NORMAL, NULL);
@@ -4918,7 +4920,9 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 		len = OPEN_ALWAYS;
 	}
 	mode = FILE_ATTRIBUTE_NORMAL;
-	utf8_to_utf16(dpath, -1, &wpath, NULL);
+	rc = utf8_to_utf16(dpath, -1, &wpath, NULL);
+	if (rc)
+		goto leave;
 	env->me_fd = CreateFileW(wpath, oflags, FILE_SHARE_READ|FILE_SHARE_WRITE,
 		NULL, len, mode, NULL);
 	free(wpath);
@@ -4950,7 +4954,9 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 			 */
 #ifdef _WIN32
 			len = OPEN_EXISTING;
-			utf8_to_utf16(dpath, -1, &wpath, NULL);
+			rc = utf8_to_utf16(dpath, -1, &wpath, NULL);
+			if (rc)
+				goto leave;
 			env->me_mfd = CreateFileW(wpath, oflags,
 				FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, len,
 				mode | FILE_FLAG_WRITE_THROUGH, NULL);
@@ -9455,7 +9461,9 @@ mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags)
 	 * already in the OS cache.
 	 */
 #ifdef _WIN32
-	utf8_to_utf16(lpath, -1, &wpath, NULL);
+	rc = utf8_to_utf16(lpath, -1, &wpath, NULL);
+	if (rc)
+		return rc;
 	newfd = CreateFileW(wpath, GENERIC_WRITE, 0, NULL, CREATE_NEW,
 				FILE_FLAG_NO_BUFFERING|FILE_FLAG_WRITE_THROUGH, NULL);
 	free(wpath);
@@ -10187,6 +10195,8 @@ static int utf8_to_utf16(const char *src, int srcsize, wchar_t **dst, int *dstsi
 	if (need == 0)
 		return EINVAL;
 	result = malloc(sizeof(wchar_t) * need);
+	if (!result)
+		return ENOMEM;
 	MultiByteToWideChar(CP_UTF8, 0, src, srcsize, result, need);
 	if (dstsize)
 		*dstsize = need;

From 2cc88d20ccc33165f658d9678f19df38d08e71d1 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sat, 19 Dec 2015 22:53:26 +0000
Subject: [PATCH 189/504] Add Getting Started doc

---
 libraries/liblmdb/Doxyfile  |   2 +-
 libraries/liblmdb/intro.doc | 192 ++++++++++++++++++++++++++++++++++++
 libraries/liblmdb/lmdb.h    |   3 +
 3 files changed, 196 insertions(+), 1 deletion(-)
 create mode 100644 libraries/liblmdb/intro.doc

diff --git a/libraries/liblmdb/Doxyfile b/libraries/liblmdb/Doxyfile
index 92d17b09eb..5047c0bb1f 100644
--- a/libraries/liblmdb/Doxyfile
+++ b/libraries/liblmdb/Doxyfile
@@ -582,7 +582,7 @@ WARN_LOGFILE           =
 # directories like "/usr/src/myproject". Separate the files or directories
 # with spaces.
 
-INPUT                  = lmdb.h midl.h mdb.c midl.c
+INPUT                  = lmdb.h midl.h mdb.c midl.c intro.doc
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is
diff --git a/libraries/liblmdb/intro.doc b/libraries/liblmdb/intro.doc
new file mode 100644
index 0000000000..f7f7b10aee
--- /dev/null
+++ b/libraries/liblmdb/intro.doc
@@ -0,0 +1,192 @@
+/*
+ * Copyright 2015 Howard Chu, Symas Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
+ */
+/** @page starting Getting Started
+
+LMDB is compact, fast, powerful, and robust and implements a simplified
+variant of the BerkeleyDB (BDB) API. (BDB is also very powerful, and verbosely
+documented in its own right.) After reading this page, the main
+\ref mdb documentation should make sense. Thanks to Bert Hubert
+for creating the
+<a href="https://github.com/ahupowerdns/ahutils/blob/master/lmdb-semantics.md">
+initial version</a> of this writeup.
+
+Everything starts with an environment, created by #mdb_env_create().
+Once created, this environment must also be opened with #mdb_env_open().
+
+#mdb_env_open() gets passed a name which is interpreted as a directory
+path. Note that this directory must exist already, it is not created
+for you. Within that directory, a lock file and a storage file will be
+generated. If you don't want to use a directory, you can pass the
+#MDB_NOSUBDIR option, in which case the path you provided is used
+directly as the data file, and another file with a "-lock" suffix
+added will be used for the lock file.
+
+Once the environment is open, a transaction can be created within it
+using #mdb_txn_begin(). Transactions may be read-write or read-only,
+and read-write transactions may be nested. A transaction must only
+be used by one thread at a time. Transactions are always required,
+even for read-only access. The transaction provides a consistent
+view of the data.
+
+Once a transaction has been created, a database can be opened within it
+using #mdb_dbi_open(). If only one database will ever be used in the
+environment, a NULL can be passed as the database name. For named
+databases, the #MDB_CREATE flag must be used to create the database
+if it doesn't already exist. Also, #mdb_env_set_maxdbs() must be
+called after #mdb_env_create() and before #mdb_env_open() to set the
+maximum number of named databases you want to support.
+
+Note: a single transaction can open multiple databases. Generally
+databases should only be opened once, by the first transaction in
+the process. After the first transaction completes, the database
+handles can freely be used by all subsequent transactions.
+
+Within a transaction, #mdb_get() and #mdb_put() can store single
+key/value pairs if that is all you need to do (but see \ref Cursors
+below if you want to do more).
+
+A key/value pair is expressed as two #MDB_val structures. This struct
+has two fields, \c mv_size and \c mv_data. The data is a \c void pointer to
+an array of \c mv_size bytes.
+
+Because LMDB is very efficient (and usually zero-copy), the data returned
+in an #MDB_val structure may be memory-mapped straight from disk. In
+other words <b>look but do not touch</b> (or free() for that matter).
+Once a transaction is closed, the values can no longer be used, so
+make a copy if you need to keep them after that.
+
+@section Cursors Cursors
+
+To do more powerful things, we must use a cursor.
+
+Within the transaction, a cursor can be created with #mdb_cursor_open().
+With this cursor we can store/retrieve/delete (multiple) values using
+#mdb_cursor_get(), #mdb_cursor_put(), and #mdb_cursor_del().
+
+#mdb_cursor_get() positions itself depending on the cursor operation
+requested, and for some operations, on the supplied key. For example,
+to list all key/value pairs in a database, use operation #MDB_FIRST for
+the first call to #mdb_cursor_get(), and #MDB_NEXT on subsequent calls,
+until the end is hit.
+
+To retrieve all keys starting from a specified key value, use #MDB_SET.
+For more cursor operations, see the \ref mdb docs.
+
+When using #mdb_cursor_put(), either the function will position the
+cursor for you based on the \b key, or you can use operation
+#MDB_CURRENT to use the current position of the cursor. Note that
+\b key must then match the current position's key.
+
+@subsection summary Summarizing the Opening
+
+So we have a cursor in a transaction which opened a database in an
+environment which is opened from a filesystem after it was
+separately created.
+
+Or, we create an environment, open it from a filesystem, create a
+transaction within it, open a database within that transaction,
+and create a cursor within all of the above.
+
+Got it?
+
+@section thrproc Threads and Processes
+
+LMDB uses POSIX locks on files, and these locks have issues if one
+process opens a file multiple times. Because of this, do not
+#mdb_env_open() a file multiple times from a single process. Instead,
+share the LMDB environment that has opened the file across all threads.
+Otherwise, if a single process opens the same environment multiple times,
+closing it once will remove all the locks held on it, and the other
+instances will be vulnerable to corruption from other processes.
+
+Also note that a transaction is tied to one thread by default using
+Thread Local Storage. If you want to pass read-only transactions across
+threads, you can use the #MDB_NOTLS option on the environment.
+
+@section txns Transactions, Rollbacks, etc.
+
+To actually get anything done, a transaction must be committed using
+#mdb_txn_commit(). Alternatively, all of a transaction's operations
+can be discarded using #mdb_txn_abort(). In a read-only transaction,
+any cursors will \b not automatically be freed. In a read-write
+transaction, all cursors will be freed and must not be used again.
+
+For read-only transactions, obviously there is nothing to commit to
+storage. The transaction still must eventually be aborted to close
+any database handle(s) opened in it, or committed to keep the
+database handles around for reuse in new transactions.
+
+In addition, as long as a transaction is open, a consistent view of
+the database is kept alive, which requires storage. A read-only
+transaction that no longer requires this consistent view should
+be terminated (committed or aborted) when the view is no longer
+needed (but see below for an optimization).
+
+There can be multiple simultaneously active read-only transactions
+but only one that can write. Once a single read-write transaction
+is opened, all further attempts to begin one will block until the
+first one is committed or aborted. This has no effect on read-only
+transactions, however, and they may contine to be opened at any time.
+
+@section dupkeys Duplicate Keys
+
+#mdb_get() and #mdb_put() respectively have no and only some support
+for multiple key/value pairs with identical keys. If there are multiple
+values for a key, #mdb_get() will only return the first value.
+
+When multiple values for one key are required, pass the #MDB_DUPSORT
+flag to #mdb_dbi_open(). In an #MDB_DUPSORT database, by default
+#mdb_put() will not replace the value for a key if the key existed
+already. Instead it will add the new value to the key. In addition,
+#mdb_del() will pay attention to the value field too, allowing for
+specific values of a key to be deleted.
+
+Finally, additional cursor operations become available for
+traversing through and retrieving duplicate values.
+
+@section optim Some Optimization
+
+If you frequently begin and abort read-only transactions, as an
+optimization, it is possible to only reset and renew a transaction.
+
+#mdb_txn_reset() releases any old copies of data kept around for
+a read-only transaction. To reuse this reset transaction, call
+#mdb_txn_renew() on it. Any cursors in this transaction must also
+be renewed using #mdb_cursor_renew().
+
+Note that #mdb_txn_reset() is similar to #mdb_txn_abort() and will
+close any databases you opened within the transaction.
+
+To permanently free a transaction, reset or not, use #mdb_txn_abort().
+
+@section cleanup Cleaning Up
+
+For read-only transactions, any cursors created within it must
+be closed using #mdb_cursor_close().
+
+It is very rarely necessary to close a database handle, and in
+general they should just be left open.
+
+@section onward The Full API
+
+The full \ref mdb documentation lists further details, like how to:
+
+  \li size a database (the default limits are intentionally small)
+  \li drop and clean a database
+  \li detect and report errors
+  \li optimize (bulk) loading speed
+  \li (temporarily) reduce robustness to gain even more speed
+  \li gather statistics about the database
+  \li define custom sort orders
+
+*/
diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index e080e76d0f..e481b34f05 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -40,6 +40,9 @@
  *	corrupt the database. Of course if your application code is known to
  *	be bug-free (...) then this is not an issue.
  *
+ *	If this is your first time using a transactional embedded key/value
+ *	store, you may find the \ref starting page to be helpful.
+ *
  *	@section caveats_sec Caveats
  *	Troubleshooting the lock file, plus semaphores on BSD systems:
  *

From 86ae31ebf51c6f242f8f19c41ff5127ca0fff1fa Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sat, 19 Dec 2015 22:57:00 +0000
Subject: [PATCH 190/504] Fix typos

---
 libraries/liblmdb/intro.doc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/intro.doc b/libraries/liblmdb/intro.doc
index f7f7b10aee..870c7bb8e7 100644
--- a/libraries/liblmdb/intro.doc
+++ b/libraries/liblmdb/intro.doc
@@ -136,7 +136,7 @@ There can be multiple simultaneously active read-only transactions
 but only one that can write. Once a single read-write transaction
 is opened, all further attempts to begin one will block until the
 first one is committed or aborted. This has no effect on read-only
-transactions, however, and they may contine to be opened at any time.
+transactions, however, and they may continue to be opened at any time.
 
 @section dupkeys Duplicate Keys
 

From 1ba5adb2ec262405f9207d6015d4f29eea548d25 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sat, 19 Dec 2015 23:51:31 +0000
Subject: [PATCH 191/504] MDB_VL32 preparation

Use 64 bit types consistently. This keeps database structures the
same size for MDB_VL32 and native 64 bit builds.
---
 libraries/liblmdb/lmdb.h     |  25 +++--
 libraries/liblmdb/mdb.c      | 206 +++++++++++++++++++----------------
 libraries/liblmdb/mdb_dump.c |  11 +-
 libraries/liblmdb/mdb_load.c |  11 +-
 libraries/liblmdb/mdb_stat.c |  25 +++--
 libraries/liblmdb/midl.h     |   9 ++
 6 files changed, 176 insertions(+), 111 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index e481b34f05..d2bf595058 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -161,6 +161,7 @@
 #define _LMDB_H_
 
 #include <sys/types.h>
+#include <inttypes.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -173,6 +174,12 @@ typedef	int	mdb_mode_t;
 typedef	mode_t	mdb_mode_t;
 #endif
 
+#ifdef MDB_VL32
+typedef uint64_t	mdb_size_t;
+#else
+typedef size_t	mdb_size_t;
+#endif
+
 /** An abstraction for a file handle.
  *	On POSIX systems file handles are small integers. On Windows
  *	they're opaque pointers.
@@ -450,18 +457,18 @@ typedef struct MDB_stat {
 	unsigned int	ms_psize;			/**< Size of a database page.
 											This is currently the same for all databases. */
 	unsigned int	ms_depth;			/**< Depth (height) of the B-tree */
-	size_t		ms_branch_pages;	/**< Number of internal (non-leaf) pages */
-	size_t		ms_leaf_pages;		/**< Number of leaf pages */
-	size_t		ms_overflow_pages;	/**< Number of overflow pages */
-	size_t		ms_entries;			/**< Number of data items */
+	mdb_size_t		ms_branch_pages;	/**< Number of internal (non-leaf) pages */
+	mdb_size_t		ms_leaf_pages;		/**< Number of leaf pages */
+	mdb_size_t		ms_overflow_pages;	/**< Number of overflow pages */
+	mdb_size_t		ms_entries;			/**< Number of data items */
 } MDB_stat;
 
 /** @brief Information about the environment */
 typedef struct MDB_envinfo {
 	void	*me_mapaddr;			/**< Address of map, if fixed */
-	size_t	me_mapsize;				/**< Size of the data memory map */
-	size_t	me_last_pgno;			/**< ID of the last used page */
-	size_t	me_last_txnid;			/**< ID of the last committed transaction */
+	mdb_size_t	me_mapsize;				/**< Size of the data memory map */
+	mdb_size_t	me_last_pgno;			/**< ID of the last used page */
+	mdb_size_t	me_last_txnid;			/**< ID of the last committed transaction */
 	unsigned int me_maxreaders;		/**< max reader slots in the environment */
 	unsigned int me_numreaders;		/**< max reader slots used in the environment */
 } MDB_envinfo;
@@ -828,7 +835,7 @@ int  mdb_env_get_fd(MDB_env *env, mdb_filehandle_t *fd);
 	 *   	an active write transaction.
 	 * </ul>
 	 */
-int  mdb_env_set_mapsize(MDB_env *env, size_t size);
+int  mdb_env_set_mapsize(MDB_env *env, mdb_size_t size);
 
 	/** @brief Set the maximum number of threads/reader slots for the environment.
 	 *
@@ -977,7 +984,7 @@ MDB_env *mdb_txn_env(MDB_txn *txn);
 	 * @param[in] txn A transaction handle returned by #mdb_txn_begin()
 	 * @return A transaction ID, valid if input is an active transaction.
 	 */
-size_t mdb_txn_id(MDB_txn *txn);
+mdb_size_t mdb_txn_id(MDB_txn *txn);
 
 	/** @brief Commit all the operations of a transaction into the database.
 	 *
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 57c6a989c6..0e190e9d13 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -35,6 +35,9 @@
 #ifndef _GNU_SOURCE
 #define _GNU_SOURCE 1
 #endif
+#ifdef MDB_VL32
+#define _FILE_OFFSET_BITS	64
+#endif
 #ifdef _WIN32
 #include <malloc.h>
 #include <windows.h>
@@ -240,7 +243,7 @@ union semun {
 #define ESECT
 #endif
 
-#ifdef _MSC_VER
+#ifdef _WIN32
 #define CALL_CONV WINAPI
 #else
 #define CALL_CONV
@@ -452,6 +455,16 @@ typedef pthread_mutex_t mdb_mutex_t[1], *mdb_mutexref_t;
 #define	GET_PAGESIZE(x)	((x) = sysconf(_SC_PAGE_SIZE))
 #endif
 
+#ifdef MDB_VL32
+#ifdef _WIN32
+#define	Y	"I64"
+#else
+#define	Y	"ll"
+#endif
+#else
+#define Y	Z
+#endif
+
 #if defined(_WIN32) || defined(MDB_USE_POSIX_SEM)
 #define MNAME_LEN	32
 #elif defined(MDB_USE_SYSV_SEM)
@@ -1062,7 +1075,7 @@ typedef struct MDB_db {
 	pgno_t		md_branch_pages;	/**< number of internal pages */
 	pgno_t		md_leaf_pages;		/**< number of leaf pages */
 	pgno_t		md_overflow_pages;	/**< number of overflow pages */
-	size_t		md_entries;		/**< number of data items */
+	pgno_t		md_entries;		/**< number of data items */
 	pgno_t		md_root;		/**< the root page of this tree */
 } MDB_db;
 
@@ -1092,8 +1105,16 @@ typedef struct MDB_meta {
 	uint32_t	mm_magic;
 		/** Version number of this file. Must be set to #MDB_DATA_VERSION. */
 	uint32_t	mm_version;
+#ifdef MDB_VL32
+	union {		/* always zero since we don't support fixed mapping in MDB_VL32 */
+		MDB_ID	mmun_ull;
+		void *mmun_address;
+	} mm_un;
+#define	mm_address mm_un.mmun_address
+#else
 	void		*mm_address;		/**< address for fixed mapping */
-	size_t		mm_mapsize;			/**< size of mmap region */
+#endif
+	pgno_t		mm_mapsize;			/**< size of mmap region */
 	MDB_db		mm_dbs[CORE_DBS];	/**< first is free space, 2nd is main db */
 	/** The size of pages used in this DB */
 #define	mm_psize	mm_dbs[FREE_DBI].md_pad
@@ -1318,7 +1339,7 @@ struct MDB_env {
 	void		*me_pbuf;		/**< scratch area for DUPSORT put() */
 	MDB_txn		*me_txn;		/**< current write transaction */
 	MDB_txn		*me_txn0;		/**< prealloc'd write transaction */
-	size_t		me_mapsize;		/**< size of the data memory map */
+	mdb_size_t	me_mapsize;		/**< size of the data memory map */
 	off_t		me_size;		/**< current file size */
 	pgno_t		me_maxpg;		/**< me_mapsize / me_psize */
 	MDB_dbx		*me_dbxs;		/**< array of static DB info */
@@ -1649,20 +1670,20 @@ mdb_page_list(MDB_page *mp)
 	case P_LEAF|P_LEAF2:        type = "LEAF2 page";		break;
 	case P_LEAF|P_LEAF2|P_SUBP: type = "LEAF2 sub-page";	break;
 	case P_OVERFLOW:
-		fprintf(stderr, "Overflow page %"Z"u pages %u%s\n",
+		fprintf(stderr, "Overflow page %"Y"u pages %u%s\n",
 			pgno, mp->mp_pages, state);
 		return;
 	case P_META:
-		fprintf(stderr, "Meta-page %"Z"u txnid %"Z"u\n",
+		fprintf(stderr, "Meta-page %"Y"u txnid %"Y"u\n",
 			pgno, ((MDB_meta *)METADATA(mp))->mm_txnid);
 		return;
 	default:
-		fprintf(stderr, "Bad page %"Z"u flags 0x%u\n", pgno, mp->mp_flags);
+		fprintf(stderr, "Bad page %"Y"u flags 0x%u\n", pgno, mp->mp_flags);
 		return;
 	}
 
 	nkeys = NUMKEYS(mp);
-	fprintf(stderr, "%s %"Z"u numkeys %d%s\n", type, pgno, nkeys, state);
+	fprintf(stderr, "%s %"Y"u numkeys %d%s\n", type, pgno, nkeys, state);
 
 	for (i=0; i<nkeys; i++) {
 		if (IS_LEAF2(mp)) {	/* LEAF2 pages have no mp_ptrs[] or node headers */
@@ -1677,7 +1698,7 @@ mdb_page_list(MDB_page *mp)
 		key.mv_data = node->mn_data;
 		nsize = NODESIZE + key.mv_size;
 		if (IS_BRANCH(mp)) {
-			fprintf(stderr, "key %d: page %"Z"u, %s\n", i, NODEPGNO(node),
+			fprintf(stderr, "key %d: page %"Y"u, %s\n", i, NODEPGNO(node),
 				DKEY(&key));
 			total += nsize;
 		} else {
@@ -1773,7 +1794,7 @@ static void mdb_audit(MDB_txn *txn)
 		}
 	}
 	if (freecount + count + NUM_METAS != txn->mt_next_pgno) {
-		fprintf(stderr, "audit: %lu freecount: %lu count: %lu total: %lu next_pgno: %lu\n",
+		fprintf(stderr, "audit: %"Y"u freecount: %"Y"u count: %"Y"u total: %"Y"u next_pgno: %"Y"u\n",
 			txn->mt_txnid, freecount, count+NUM_METAS,
 			freecount+count+NUM_METAS, txn->mt_next_pgno);
 	}
@@ -1790,8 +1811,8 @@ int
 mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b)
 {
 	MDB_cmp_func *dcmp = txn->mt_dbxs[dbi].md_dcmp;
-#if UINT_MAX < SIZE_MAX
-	if (dcmp == mdb_cmp_int && a->mv_size == sizeof(size_t))
+#if UINT_MAX < SIZE_MAX || defined(MDB_VL32)
+	if (dcmp == mdb_cmp_int && a->mv_size == sizeof(mdb_size_t))
 		dcmp = mdb_cmp_clong;
 #endif
 	return dcmp(a, b);
@@ -1914,7 +1935,7 @@ mdb_page_loose(MDB_cursor *mc, MDB_page *mp)
 		}
 	}
 	if (loose) {
-		DPRINTF(("loosen db %d page %"Z"u", DDBI(mc),
+		DPRINTF(("loosen db %d page %"Y"u", DDBI(mc),
 			mp->mp_pgno));
 		NEXT_LOOSE_PAGE(mp) = txn->mt_loose_pgs;
 		txn->mt_loose_pgs = mp;
@@ -2210,7 +2231,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
 		np = txn->mt_loose_pgs;
 		txn->mt_loose_pgs = NEXT_LOOSE_PAGE(np);
 		txn->mt_loose_count--;
-		DPRINTF(("db %d use loose page %"Z"u", DDBI(mc),
+		DPRINTF(("db %d use loose page %"Y"u", DDBI(mc),
 				np->mp_pgno));
 		*mp = np;
 		return MDB_SUCCESS;
@@ -2305,10 +2326,10 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
 		}
 		env->me_pglast = last;
 #if (MDB_DEBUG) > 1
-		DPRINTF(("IDL read txn %"Z"u root %"Z"u num %u",
+		DPRINTF(("IDL read txn %"Y"u root %"Y"u num %u",
 			last, txn->mt_dbs[FREE_DBI].md_root, i));
 		for (j = i; j; j--)
-			DPRINTF(("IDL %"Z"u", idl[j]));
+			DPRINTF(("IDL %"Y"u", idl[j]));
 #endif
 		/* Merge in descending sorted order */
 		mdb_midl_xmerge(mop, idl);
@@ -2478,7 +2499,7 @@ mdb_page_touch(MDB_cursor *mc)
 			(rc = mdb_page_alloc(mc, 1, &np)))
 			goto fail;
 		pgno = np->mp_pgno;
-		DPRINTF(("touched db %d page %"Z"u -> %"Z"u", DDBI(mc),
+		DPRINTF(("touched db %d page %"Y"u -> %"Y"u", DDBI(mc),
 			mp->mp_pgno, pgno));
 		mdb_cassert(mc, mp->mp_pgno != pgno);
 		mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno);
@@ -2868,7 +2889,7 @@ mdb_txn_renew(MDB_txn *txn)
 
 	rc = mdb_txn_renew0(txn);
 	if (rc == MDB_SUCCESS) {
-		DPRINTF(("renew txn %"Z"u%c %p on mdbenv %p, root page %"Z"u",
+		DPRINTF(("renew txn %"Y"u%c %p on mdbenv %p, root page %"Y"u",
 			txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w',
 			(void *)txn, (void *)txn->mt_env, txn->mt_dbs[MAIN_DBI].md_root));
 	}
@@ -2968,7 +2989,7 @@ renew:
 	} else {
 		txn->mt_flags |= flags;	/* could not change txn=me_txn0 earlier */
 		*ret = txn;
-		DPRINTF(("begin txn %"Z"u%c %p on mdbenv %p, root page %"Z"u",
+		DPRINTF(("begin txn %"Y"u%c %p on mdbenv %p, root page %"Y"u",
 			txn->mt_txnid, (flags & MDB_RDONLY) ? 'r' : 'w',
 			(void *) txn, (void *) env, txn->mt_dbs[MAIN_DBI].md_root));
 	}
@@ -2983,7 +3004,7 @@ mdb_txn_env(MDB_txn *txn)
 	return txn->mt_env;
 }
 
-size_t
+mdb_size_t
 mdb_txn_id(MDB_txn *txn)
 {
     if(!txn) return 0;
@@ -3035,7 +3056,7 @@ mdb_txn_end(MDB_txn *txn, unsigned mode)
 	/* Export or close DBI handles opened in this txn */
 	mdb_dbis_update(txn, mode & MDB_END_UPDATE);
 
-	DPRINTF(("%s txn %"Z"u%c %p on mdbenv %p, root page %"Z"u",
+	DPRINTF(("%s txn %"Y"u%c %p on mdbenv %p, root page %"Y"u",
 		names[mode & MDB_END_OPMASK],
 		txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w',
 		(void *) txn, (void *)env, txn->mt_dbs[MAIN_DBI].md_root));
@@ -3209,10 +3230,10 @@ mdb_freelist_save(MDB_txn *txn)
 #if (MDB_DEBUG) > 1
 			{
 				unsigned int i = free_pgs[0];
-				DPRINTF(("IDL write txn %"Z"u root %"Z"u num %u",
+				DPRINTF(("IDL write txn %"Y"u root %"Y"u num %u",
 					txn->mt_txnid, txn->mt_dbs[FREE_DBI].md_root, i));
 				for (; i; i--)
-					DPRINTF(("IDL %"Z"u", free_pgs[i]));
+					DPRINTF(("IDL %"Y"u", free_pgs[i]));
 			}
 #endif
 			continue;
@@ -3323,15 +3344,16 @@ mdb_page_flush(MDB_txn *txn, int keep)
 	MDB_ID2L	dl = txn->mt_u.dirty_list;
 	unsigned	psize = env->me_psize, j;
 	int			i, pagecount = dl[0].mid, rc;
-	size_t		size = 0, pos = 0;
+	size_t		size = 0;
+	off_t		pos = 0;
 	pgno_t		pgno = 0;
 	MDB_page	*dp = NULL;
 #ifdef _WIN32
 	OVERLAPPED	ov;
 #else
 	struct iovec iov[MDB_COMMIT_PAGES];
-	ssize_t		wpos = 0, wsize = 0, wres;
-	size_t		next_pos = 1; /* impossible pos, so pos != next_pos */
+	ssize_t		wsize = 0, wres;
+	off_t		wpos = 0, next_pos = 1; /* impossible pos, so pos != next_pos */
 	int			n = 0;
 #endif
 
@@ -3430,7 +3452,7 @@ retry_seek:
 			wpos = pos;
 			wsize = 0;
 		}
-		DPRINTF(("committing page %"Z"u", pgno));
+		DPRINTF(("committing page %"Y"u", pgno));
 		next_pos = pos + size;
 		iov[n].iov_len = size;
 		iov[n].iov_base = (char *)dp;
@@ -3639,7 +3661,7 @@ mdb_txn_commit(MDB_txn *txn)
 		!(txn->mt_flags & (MDB_TXN_DIRTY|MDB_TXN_SPILLS)))
 		goto done;
 
-	DPRINTF(("committing txn %"Z"u %p on mdbenv %p, root page %"Z"u",
+	DPRINTF(("committing txn %"Y"u %p on mdbenv %p, root page %"Y"u",
 	    txn->mt_txnid, (void*)txn, (void*)env, txn->mt_dbs[MAIN_DBI].md_root));
 
 	/* Update DB root pointers */
@@ -3737,7 +3759,7 @@ mdb_env_read_header(MDB_env *env, MDB_meta *meta)
 		p = (MDB_page *)&pbuf;
 
 		if (!F_ISSET(p->mp_flags, P_META)) {
-			DPRINTF(("page %"Z"u not a meta page", p->mp_pgno));
+			DPRINTF(("page %"Y"u not a meta page", p->mp_pgno));
 			return MDB_INVALID;
 		}
 
@@ -3837,7 +3859,7 @@ mdb_env_write_meta(MDB_txn *txn)
 	MDB_env *env;
 	MDB_meta	meta, metab, *mp;
 	unsigned flags;
-	size_t mapsize;
+	mdb_size_t mapsize;
 	off_t off;
 	int rc, len, toggle;
 	char *ptr;
@@ -3849,7 +3871,7 @@ mdb_env_write_meta(MDB_txn *txn)
 #endif
 
 	toggle = txn->mt_txnid & 1;
-	DPRINTF(("writing meta page %d for root page %"Z"u",
+	DPRINTF(("writing meta page %d for root page %"Y"u",
 		toggle, txn->mt_dbs[MAIN_DBI].md_root));
 
 	env = txn->mt_env;
@@ -4003,7 +4025,7 @@ mdb_env_map(MDB_env *env, void *addr)
 	int access = SECTION_MAP_READ;
 	HANDLE mh;
 	void *map;
-	size_t msize;
+	SIZE_T msize;
 	ULONG pageprot = PAGE_READONLY, secprot, alloctype;
 
 	if (flags & MDB_WRITEMAP) {
@@ -4071,7 +4093,7 @@ mdb_env_map(MDB_env *env, void *addr)
 }
 
 int ESECT
-mdb_env_set_mapsize(MDB_env *env, size_t size)
+mdb_env_set_mapsize(MDB_env *env, mdb_size_t size)
 {
 	/* If env is already open, caller is responsible for making
 	 * sure there are no active txns.
@@ -4132,7 +4154,7 @@ mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers)
 }
 
 static int ESECT
-mdb_fsize(HANDLE fd, size_t *size)
+mdb_fsize(HANDLE fd, mdb_size_t *size)
 {
 #ifdef _WIN32
 	LARGE_INTEGER fsize;
@@ -4246,7 +4268,7 @@ mdb_env_open2(MDB_env *env)
 		/* Make sure mapsize >= committed data size.  Even when using
 		 * mm_mapsize, which could be broken in old files (ITS#7789).
 		 */
-		size_t minsize = (meta.mm_last_pg + 1) * meta.mm_psize;
+		mdb_size_t minsize = (meta.mm_last_pg + 1) * meta.mm_psize;
 		if (env->me_mapsize < minsize)
 			env->me_mapsize = minsize;
 	}
@@ -4308,11 +4330,11 @@ mdb_env_open2(MDB_env *env)
 			meta->mm_version, env->me_psize));
 		DPRINTF(("using meta page %d",    (int) (meta->mm_txnid & 1)));
 		DPRINTF(("depth: %u",             db->md_depth));
-		DPRINTF(("entries: %"Z"u",        db->md_entries));
-		DPRINTF(("branch pages: %"Z"u",   db->md_branch_pages));
-		DPRINTF(("leaf pages: %"Z"u",     db->md_leaf_pages));
-		DPRINTF(("overflow pages: %"Z"u", db->md_overflow_pages));
-		DPRINTF(("root: %"Z"u",           db->md_root));
+		DPRINTF(("entries: %"Y"u",        db->md_entries));
+		DPRINTF(("branch pages: %"Y"u",   db->md_branch_pages));
+		DPRINTF(("leaf pages: %"Y"u",     db->md_leaf_pages));
+		DPRINTF(("overflow pages: %"Y"u", db->md_overflow_pages));
+		DPRINTF(("root: %"Y"u",           db->md_root));
 	}
 #endif
 
@@ -5131,18 +5153,18 @@ mdb_env_close(MDB_env *env)
 	free(env);
 }
 
-/** Compare two items pointing at aligned size_t's */
+/** Compare two items pointing at aligned mdb_size_t's */
 static int
 mdb_cmp_long(const MDB_val *a, const MDB_val *b)
 {
-	return (*(size_t *)a->mv_data < *(size_t *)b->mv_data) ? -1 :
-		*(size_t *)a->mv_data > *(size_t *)b->mv_data;
+	return (*(mdb_size_t *)a->mv_data < *(mdb_size_t *)b->mv_data) ? -1 :
+		*(mdb_size_t *)a->mv_data > *(mdb_size_t *)b->mv_data;
 }
 
 /** Compare two items pointing at aligned unsigned int's.
  *
  *	This is also set as #MDB_INTEGERDUP|#MDB_DUPFIXED's #MDB_dbx.%md_dcmp,
- *	but #mdb_cmp_clong() is called instead if the data type is size_t.
+ *	but #mdb_cmp_clong() is called instead if the data type is mdb_size_t.
  */
 static int
 mdb_cmp_int(const MDB_val *a, const MDB_val *b)
@@ -5247,7 +5269,7 @@ mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp)
 
 	nkeys = NUMKEYS(mp);
 
-	DPRINTF(("searching %u keys in %s %spage %"Z"u",
+	DPRINTF(("searching %u keys in %s %spage %"Y"u",
 	    nkeys, IS_LEAF(mp) ? "leaf" : "branch", IS_SUBP(mp) ? "sub-" : "",
 	    mdb_dbg_pgno(mp)));
 
@@ -5259,7 +5281,7 @@ mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp)
 	 * alignment is guaranteed. Use faster mdb_cmp_int.
 	 */
 	if (cmp == mdb_cmp_cint && IS_BRANCH(mp)) {
-		if (NODEPTR(mp, 1)->mn_ksize == sizeof(size_t))
+		if (NODEPTR(mp, 1)->mn_ksize == sizeof(mdb_size_t))
 			cmp = mdb_cmp_long;
 		else
 			cmp = mdb_cmp_int;
@@ -5295,7 +5317,7 @@ mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp)
 				DPRINTF(("found leaf index %u [%s], rc = %i",
 				    i, DKEY(&nodekey), rc));
 			else
-				DPRINTF(("found branch index %u [%s -> %"Z"u], rc = %i",
+				DPRINTF(("found branch index %u [%s -> %"Y"u], rc = %i",
 				    i, DKEY(&nodekey), NODEPGNO(node), rc));
 #endif
 			if (rc == 0)
@@ -5343,7 +5365,7 @@ static void
 mdb_cursor_pop(MDB_cursor *mc)
 {
 	if (mc->mc_snum) {
-		DPRINTF(("popping page %"Z"u off db %d cursor %p",
+		DPRINTF(("popping page %"Y"u off db %d cursor %p",
 			mc->mc_pg[mc->mc_top]->mp_pgno, DDBI(mc), (void *) mc));
 
 		mc->mc_snum--;
@@ -5359,7 +5381,7 @@ mdb_cursor_pop(MDB_cursor *mc)
 static int
 mdb_cursor_push(MDB_cursor *mc, MDB_page *mp)
 {
-	DPRINTF(("pushing page %"Z"u on db %d cursor %p", mp->mp_pgno,
+	DPRINTF(("pushing page %"Y"u on db %d cursor %p", mp->mp_pgno,
 		DDBI(mc), (void *) mc));
 
 	if (mc->mc_snum >= CURSOR_STACK) {
@@ -5422,7 +5444,7 @@ mdb_page_get(MDB_txn *txn, pgno_t pgno, MDB_page **ret, int *lvl)
 		level = 0;
 		p = (MDB_page *)(env->me_map + env->me_psize * pgno);
 	} else {
-		DPRINTF(("page %"Z"u not found", pgno));
+		DPRINTF(("page %"Y"u not found", pgno));
 		txn->mt_flags |= MDB_TXN_ERROR;
 		return MDB_PAGE_NOTFOUND;
 	}
@@ -5448,13 +5470,13 @@ mdb_page_search_root(MDB_cursor *mc, MDB_val *key, int flags)
 		MDB_node	*node;
 		indx_t		i;
 
-		DPRINTF(("branch page %"Z"u has %u keys", mp->mp_pgno, NUMKEYS(mp)));
+		DPRINTF(("branch page %"Y"u has %u keys", mp->mp_pgno, NUMKEYS(mp)));
 		/* Don't assert on branch pages in the FreeDB. We can get here
 		 * while in the process of rebalancing a FreeDB branch page; we must
 		 * let that proceed. ITS#8336
 		 */
 		mdb_cassert(mc, !mc->mc_dbi || NUMKEYS(mp) > 1);
-		DPRINTF(("found index 0 to page %"Z"u", NODEPGNO(NODEPTR(mp, 0))));
+		DPRINTF(("found index 0 to page %"Y"u", NODEPGNO(NODEPTR(mp, 0))));
 
 		if (flags & (MDB_PS_FIRST|MDB_PS_LAST)) {
 			i = 0;
@@ -5499,7 +5521,7 @@ mdb_page_search_root(MDB_cursor *mc, MDB_val *key, int flags)
 		return MDB_CORRUPTED;
 	}
 
-	DPRINTF(("found leaf page %"Z"u for key [%s]", mp->mp_pgno,
+	DPRINTF(("found leaf page %"Y"u for key [%s]", mp->mp_pgno,
 	    key ? DKEY(key) : "null"));
 	mc->mc_flags |= C_INITIALIZED;
 	mc->mc_flags &= ~C_EOF;
@@ -5602,7 +5624,7 @@ mdb_page_search(MDB_cursor *mc, MDB_val *key, int flags)
 	mc->mc_snum = 1;
 	mc->mc_top = 0;
 
-	DPRINTF(("db %d root page %"Z"u has flags 0x%X",
+	DPRINTF(("db %d root page %"Y"u has flags 0x%X",
 		DDBI(mc), root, mc->mc_pg[0]->mp_flags));
 
 	if (flags & MDB_PS_MODIFY) {
@@ -5627,7 +5649,7 @@ mdb_ovpage_free(MDB_cursor *mc, MDB_page *mp)
 	MDB_ID pn = pg << 1;
 	int rc;
 
-	DPRINTF(("free ov page %"Z"u (%d)", pg, ovpages));
+	DPRINTF(("free ov page %"Y"u (%d)", pg, ovpages));
 	/* If the page is dirty or on the spill list we just acquired it,
 	 * so we should give it back to our current free list, if any.
 	 * Otherwise put it onto the list of pages we freed in this txn.
@@ -5716,7 +5738,7 @@ mdb_node_read(MDB_txn *txn, MDB_node *leaf, MDB_val *data)
 	data->mv_size = NODEDSZ(leaf);
 	memcpy(&pgno, NODEDATA(leaf), sizeof(pgno));
 	if ((rc = mdb_page_get(txn, pgno, &omp, NULL)) != 0) {
-		DPRINTF(("read overflow page %"Z"u failed", pgno));
+		DPRINTF(("read overflow page %"Y"u failed", pgno));
 		return rc;
 	}
 	data->mv_data = METADATA(omp);
@@ -5765,7 +5787,7 @@ mdb_cursor_sibling(MDB_cursor *mc, int move_right)
 	}
 
 	mdb_cursor_pop(mc);
-	DPRINTF(("parent page is page %"Z"u, index %u",
+	DPRINTF(("parent page is page %"Y"u, index %u",
 		mc->mc_pg[mc->mc_top]->mp_pgno, mc->mc_ki[mc->mc_top]));
 
 	if (move_right ? (mc->mc_ki[mc->mc_top] + 1u >= NUMKEYS(mc->mc_pg[mc->mc_top]))
@@ -5836,7 +5858,7 @@ mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
 		}
 	}
 
-	DPRINTF(("cursor_next: top page is %"Z"u in cursor %p",
+	DPRINTF(("cursor_next: top page is %"Y"u in cursor %p",
 		mdb_dbg_pgno(mp), (void *) mc));
 	if (mc->mc_flags & C_DEL)
 		goto skip;
@@ -5848,12 +5870,12 @@ mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
 			return rc;
 		}
 		mp = mc->mc_pg[mc->mc_top];
-		DPRINTF(("next page is %"Z"u, key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top]));
+		DPRINTF(("next page is %"Y"u, key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top]));
 	} else
 		mc->mc_ki[mc->mc_top]++;
 
 skip:
-	DPRINTF(("==> cursor points to page %"Z"u with %u keys, key index %u",
+	DPRINTF(("==> cursor points to page %"Y"u with %u keys, key index %u",
 	    mdb_dbg_pgno(mp), NUMKEYS(mp), mc->mc_ki[mc->mc_top]));
 
 	if (IS_LEAF2(mp)) {
@@ -5915,7 +5937,7 @@ mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
 		}
 	}
 
-	DPRINTF(("cursor_prev: top page is %"Z"u in cursor %p",
+	DPRINTF(("cursor_prev: top page is %"Y"u in cursor %p",
 		mdb_dbg_pgno(mp), (void *) mc));
 
 	if (mc->mc_ki[mc->mc_top] == 0)  {
@@ -5925,13 +5947,13 @@ mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
 		}
 		mp = mc->mc_pg[mc->mc_top];
 		mc->mc_ki[mc->mc_top] = NUMKEYS(mp) - 1;
-		DPRINTF(("prev page is %"Z"u, key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top]));
+		DPRINTF(("prev page is %"Y"u, key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top]));
 	} else
 		mc->mc_ki[mc->mc_top]--;
 
 	mc->mc_flags &= ~C_EOF;
 
-	DPRINTF(("==> cursor points to page %"Z"u with %u keys, key index %u",
+	DPRINTF(("==> cursor points to page %"Y"u with %u keys, key index %u",
 	    mdb_dbg_pgno(mp), NUMKEYS(mp), mc->mc_ki[mc->mc_top]));
 
 	if (IS_LEAF2(mp)) {
@@ -6132,8 +6154,8 @@ set1:
 			if ((rc = mdb_node_read(mc->mc_txn, leaf, &olddata)) != MDB_SUCCESS)
 				return rc;
 			dcmp = mc->mc_dbx->md_dcmp;
-#if UINT_MAX < SIZE_MAX
-			if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t))
+#if UINT_MAX < SIZE_MAX || defined(MDB_VL32)
+			if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(mdb_size_t))
 				dcmp = mdb_cmp_clong;
 #endif
 			rc = dcmp(data, &olddata);
@@ -6655,8 +6677,8 @@ more:
 				if (flags == MDB_CURRENT)
 					goto current;
 				dcmp = mc->mc_dbx->md_dcmp;
-#if UINT_MAX < SIZE_MAX
-				if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t))
+#if UINT_MAX < SIZE_MAX || defined(MDB_VL32)
+				if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(mdb_size_t))
 					dcmp = mdb_cmp_clong;
 #endif
 				/* does data match? */
@@ -7107,7 +7129,7 @@ mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp)
 
 	if ((rc = mdb_page_alloc(mc, num, &np)))
 		return rc;
-	DPRINTF(("allocated new mpage %"Z"u, page size %u",
+	DPRINTF(("allocated new mpage %"Y"u, page size %u",
 	    np->mp_pgno, mc->mc_txn->mt_env->me_psize));
 	np->mp_flags = flags | P_DIRTY;
 	np->mp_lower = (PAGEHDRSZ-PAGEBASE);
@@ -7207,7 +7229,7 @@ mdb_node_add(MDB_cursor *mc, indx_t indx,
 
 	mdb_cassert(mc, mp->mp_upper >= mp->mp_lower);
 
-	DPRINTF(("add to %s %spage %"Z"u index %i, data size %"Z"u key size %"Z"u [%s]",
+	DPRINTF(("add to %s %spage %"Y"u index %i, data size %"Z"u key size %"Z"u [%s]",
 	    IS_LEAF(mp) ? "leaf" : "branch",
 		IS_SUBP(mp) ? "sub-" : "",
 		mdb_dbg_pgno(mp), indx, data ? data->mv_size : 0,
@@ -7248,7 +7270,7 @@ mdb_node_add(MDB_cursor *mc, indx_t indx,
 				goto full;
 			if ((rc = mdb_page_new(mc, P_OVERFLOW, ovpages, &ofp)))
 				return rc;
-			DPRINTF(("allocated overflow page %"Z"u", ofp->mp_pgno));
+			DPRINTF(("allocated overflow page %"Y"u", ofp->mp_pgno));
 			flags |= F_BIGDATA;
 			goto update;
 		} else {
@@ -7305,7 +7327,7 @@ update:
 	return MDB_SUCCESS;
 
 full:
-	DPRINTF(("not enough room in page %"Z"u, got %u ptrs",
+	DPRINTF(("not enough room in page %"Y"u, got %u ptrs",
 		mdb_dbg_pgno(mp), NUMKEYS(mp)));
 	DPRINTF(("upper-lower = %u - %u = %"Z"d", mp->mp_upper,mp->mp_lower,room));
 	DPRINTF(("node size = %"Z"u", node_size));
@@ -7328,7 +7350,7 @@ mdb_node_del(MDB_cursor *mc, int ksize)
 	MDB_node	*node;
 	char		*base;
 
-	DPRINTF(("delete node %u on %s page %"Z"u", indx,
+	DPRINTF(("delete node %u on %s page %"Y"u", indx,
 	    IS_LEAF(mp) ? "leaf" : "branch", mdb_dbg_pgno(mp)));
 	numkeys = NUMKEYS(mp);
 	mdb_cassert(mc, indx < numkeys);
@@ -7484,11 +7506,11 @@ mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node)
 				mx->mx_db.md_flags |= MDB_INTEGERKEY;
 		}
 	}
-	DPRINTF(("Sub-db -%u root page %"Z"u", mx->mx_cursor.mc_dbi,
+	DPRINTF(("Sub-db -%u root page %"Y"u", mx->mx_cursor.mc_dbi,
 		mx->mx_db.md_root));
 	mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DIRTY; /* DB_DIRTY guides mdb_cursor_touch */
-#if UINT_MAX < SIZE_MAX
-	if (mx->mx_dbx.md_cmp == mdb_cmp_int && mx->mx_db.md_pad == sizeof(size_t))
+#if UINT_MAX < SIZE_MAX || defined(MDB_VL32)
+	if (mx->mx_dbx.md_cmp == mdb_cmp_int && mx->mx_db.md_pad == sizeof(mdb_size_t))
 		mx->mx_dbx.md_cmp = mdb_cmp_clong;
 #endif
 }
@@ -7521,7 +7543,7 @@ mdb_xcursor_init2(MDB_cursor *mc, MDB_xcursor *src_mx, int new_dupdata)
 	}
 	mx->mx_db = src_mx->mx_db;
 	mx->mx_cursor.mc_pg[0] = src_mx->mx_cursor.mc_pg[0];
-	DPRINTF(("Sub-db -%u root page %"Z"u", mx->mx_cursor.mc_dbi,
+	DPRINTF(("Sub-db -%u root page %"Y"u", mx->mx_cursor.mc_dbi,
 		mx->mx_db.md_root));
 }
 
@@ -7690,7 +7712,7 @@ mdb_update_key(MDB_cursor *mc, MDB_val *key)
 		char kbuf2[DKBUF_MAXKEYSIZE*2+1];
 		k2.mv_data = NODEKEY(node);
 		k2.mv_size = node->mn_ksize;
-		DPRINTF(("update key %u (ofs %u) [%s] to [%s] on page %"Z"u",
+		DPRINTF(("update key %u (ofs %u) [%s] to [%s] on page %"Y"u",
 			indx, ptr,
 			mdb_dkey(&k2, kbuf2),
 			DKEY(key),
@@ -7838,7 +7860,7 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft)
 			return rc;
 	}
 
-	DPRINTF(("moving %s node %u [%s] on page %"Z"u to node %u on page %"Z"u",
+	DPRINTF(("moving %s node %u [%s] on page %"Y"u to node %u on page %"Y"u",
 	    IS_LEAF(csrc->mc_pg[csrc->mc_top]) ? "leaf" : "branch",
 	    csrc->mc_ki[csrc->mc_top],
 		DKEY(&key),
@@ -7932,7 +7954,7 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft)
 				key.mv_size = NODEKSZ(srcnode);
 				key.mv_data = NODEKEY(srcnode);
 			}
-			DPRINTF(("update separator for source page %"Z"u to [%s]",
+			DPRINTF(("update separator for source page %"Y"u to [%s]",
 				csrc->mc_pg[csrc->mc_top]->mp_pgno, DKEY(&key)));
 			mdb_cursor_copy(csrc, &mn);
 			mn.mc_snum--;
@@ -7963,7 +7985,7 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft)
 				key.mv_size = NODEKSZ(srcnode);
 				key.mv_data = NODEKEY(srcnode);
 			}
-			DPRINTF(("update separator for destination page %"Z"u to [%s]",
+			DPRINTF(("update separator for destination page %"Y"u to [%s]",
 				cdst->mc_pg[cdst->mc_top]->mp_pgno, DKEY(&key)));
 			mdb_cursor_copy(cdst, &mn);
 			mn.mc_snum--;
@@ -8009,7 +8031,7 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst)
 	psrc = csrc->mc_pg[csrc->mc_top];
 	pdst = cdst->mc_pg[cdst->mc_top];
 
-	DPRINTF(("merging page %"Z"u into %"Z"u", psrc->mp_pgno, pdst->mp_pgno));
+	DPRINTF(("merging page %"Y"u into %"Y"u", psrc->mp_pgno, pdst->mp_pgno));
 
 	mdb_cassert(csrc, csrc->mc_snum > 1);	/* can't merge root page */
 	mdb_cassert(csrc, cdst->mc_snum > 1);
@@ -8066,7 +8088,7 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst)
 		}
 	}
 
-	DPRINTF(("dst page %"Z"u now has %u keys (%.1f%% filled)",
+	DPRINTF(("dst page %"Y"u now has %u keys (%.1f%% filled)",
 	    pdst->mp_pgno, NUMKEYS(pdst),
 		(float)PAGEFILL(cdst->mc_txn->mt_env, pdst) / 10));
 
@@ -8182,14 +8204,14 @@ mdb_rebalance(MDB_cursor *mc)
 		minkeys = 1;
 		thresh = FILL_THRESHOLD;
 	}
-	DPRINTF(("rebalancing %s page %"Z"u (has %u keys, %.1f%% full)",
+	DPRINTF(("rebalancing %s page %"Y"u (has %u keys, %.1f%% full)",
 	    IS_LEAF(mc->mc_pg[mc->mc_top]) ? "leaf" : "branch",
 	    mdb_dbg_pgno(mc->mc_pg[mc->mc_top]), NUMKEYS(mc->mc_pg[mc->mc_top]),
 		(float)PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) / 10));
 
 	if (PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) >= thresh &&
 		NUMKEYS(mc->mc_pg[mc->mc_top]) >= minkeys) {
-		DPRINTF(("no need to rebalance page %"Z"u, above fill threshold",
+		DPRINTF(("no need to rebalance page %"Y"u, above fill threshold",
 		    mdb_dbg_pgno(mc->mc_pg[mc->mc_top])));
 		return MDB_SUCCESS;
 	}
@@ -8318,7 +8340,7 @@ mdb_rebalance(MDB_cursor *mc)
 		fromleft = 1;
 	}
 
-	DPRINTF(("found neighbor page %"Z"u (%u keys, %.1f%% full)",
+	DPRINTF(("found neighbor page %"Y"u (%u keys, %.1f%% full)",
 	    mn.mc_pg[mn.mc_top]->mp_pgno, NUMKEYS(mn.mc_pg[mn.mc_top]),
 		(float)PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) / 10));
 
@@ -8520,7 +8542,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 	newindx = mc->mc_ki[mc->mc_top];
 	nkeys = NUMKEYS(mp);
 
-	DPRINTF(("-----> splitting %s page %"Z"u and adding [%s] at index %i/%i",
+	DPRINTF(("-----> splitting %s page %"Y"u and adding [%s] at index %i/%i",
 	    IS_LEAF(mp) ? "leaf" : "branch", mp->mp_pgno,
 	    DKEY(newkey), mc->mc_ki[mc->mc_top], nkeys));
 
@@ -8528,7 +8550,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 	if ((rc = mdb_page_new(mc, mp->mp_flags, 1, &rp)))
 		return rc;
 	rp->mp_pad = mp->mp_pad;
-	DPRINTF(("new right sibling: page %"Z"u", rp->mp_pgno));
+	DPRINTF(("new right sibling: page %"Y"u", rp->mp_pgno));
 
 	/* Usually when splitting the root page, the cursor
 	 * height is 1. But when called from mdb_update_key,
@@ -8546,7 +8568,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 		mc->mc_pg[0] = pp;
 		mc->mc_ki[0] = 0;
 		mc->mc_db->md_root = pp->mp_pgno;
-		DPRINTF(("root split! new root = %"Z"u", pp->mp_pgno));
+		DPRINTF(("root split! new root = %"Y"u", pp->mp_pgno));
 		new_root = mc->mc_db->md_depth++;
 
 		/* Add left (implicit) pointer. */
@@ -8563,7 +8585,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 		ptop = 0;
 	} else {
 		ptop = mc->mc_top-1;
-		DPRINTF(("parent branch page is %"Z"u", mc->mc_pg[ptop]->mp_pgno));
+		DPRINTF(("parent branch page is %"Y"u", mc->mc_pg[ptop]->mp_pgno));
 	}
 
 	mdb_cursor_copy(mc, &mn);
@@ -9388,7 +9410,7 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd)
 
 	w2 = txn->mt_next_pgno * env->me_psize;
 	{
-		size_t fsize = 0;
+		mdb_size_t fsize = 0;
 		if ((rc = mdb_fsize(env->me_fd, &fsize)))
 			goto leave;
 		if (w2 > fsize)
@@ -10010,7 +10032,7 @@ mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx)
 		if (mr[i].mr_pid) {
 			txnid_t	txnid = mr[i].mr_txnid;
 			sprintf(buf, txnid == (txnid_t)-1 ?
-				"%10d %"Z"x -\n" : "%10d %"Z"x %"Z"u\n",
+				"%10d %"Z"x -\n" : "%10d %"Z"x %"Y"u\n",
 				(int)mr[i].mr_pid, (size_t)mr[i].mr_tid, txnid);
 			if (first) {
 				first = 0;
@@ -10115,7 +10137,7 @@ mdb_reader_check0(MDB_env *env, int rlocked, int *dead)
 					}
 					for (; j<rdrs; j++)
 							if (mr[j].mr_pid == pid) {
-								DPRINTF(("clear stale reader pid %u txn %"Z"d",
+								DPRINTF(("clear stale reader pid %u txn %"Y"d",
 									(unsigned) pid, mr[j].mr_txnid));
 								mr[j].mr_pid = 0;
 								count++;
diff --git a/libraries/liblmdb/mdb_dump.c b/libraries/liblmdb/mdb_dump.c
index 16c0aae1b7..72a469052d 100644
--- a/libraries/liblmdb/mdb_dump.c
+++ b/libraries/liblmdb/mdb_dump.c
@@ -25,6 +25,15 @@
 #else
 #define Z	"z"
 #endif
+#ifdef MDB_VL32
+#ifdef _WIN32
+#define	Y	"I64"
+#else
+#define	Y	"ll"
+#endif
+#else
+#define Y	Z
+#endif
 
 #define PRINT	1
 static int mode;
@@ -115,7 +124,7 @@ static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name)
 	if (name)
 		printf("database=%s\n", name);
 	printf("type=btree\n");
-	printf("mapsize=%" Z "u\n", info.me_mapsize);
+	printf("mapsize=%" Y "u\n", info.me_mapsize);
 	if (info.me_mapaddr)
 		printf("mapaddr=%p\n", info.me_mapaddr);
 	printf("maxreaders=%u\n", info.me_maxreaders);
diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c
index d2f0968188..d1fda4bf59 100644
--- a/libraries/liblmdb/mdb_load.c
+++ b/libraries/liblmdb/mdb_load.c
@@ -43,6 +43,15 @@ static MDB_val kbuf, dbuf;
 #else
 #define Z	"z"
 #endif
+#ifdef MDB_VL32
+#ifdef _WIN32
+#define	Y	"I64"
+#else
+#define	Y	"ll"
+#endif
+#else
+#define Y	Z
+#endif
 
 #define STRLENOF(s)	(sizeof(s)-1)
 
@@ -112,7 +121,7 @@ static void readhdr(void)
 			int i;
 			ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
 			if (ptr) *ptr = '\0';
-			i = sscanf((char *)dbuf.mv_data+STRLENOF("mapsize="), "%" Z "u", &info.me_mapsize);
+			i = sscanf((char *)dbuf.mv_data+STRLENOF("mapsize="), "%" Y "u", &info.me_mapsize);
 			if (i != 1) {
 				fprintf(stderr, "%s: line %" Z "d: invalid mapsize %s\n",
 					prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapsize="));
diff --git a/libraries/liblmdb/mdb_stat.c b/libraries/liblmdb/mdb_stat.c
index 6f4b3ee4f3..b785e7acf0 100644
--- a/libraries/liblmdb/mdb_stat.c
+++ b/libraries/liblmdb/mdb_stat.c
@@ -22,6 +22,15 @@
 #else
 #define	Z	"z"
 #endif
+#ifdef MDB_VL32
+#ifdef _WIN32
+#define	Y	"I64"
+#else
+#define	Y	"ll"
+#endif
+#else
+#define Y	Z
+#endif
 
 static void prstat(MDB_stat *ms)
 {
@@ -29,10 +38,10 @@ static void prstat(MDB_stat *ms)
 	printf("  Page size: %u\n", ms->ms_psize);
 #endif
 	printf("  Tree depth: %u\n", ms->ms_depth);
-	printf("  Branch pages: %"Z"u\n", ms->ms_branch_pages);
-	printf("  Leaf pages: %"Z"u\n", ms->ms_leaf_pages);
-	printf("  Overflow pages: %"Z"u\n", ms->ms_overflow_pages);
-	printf("  Entries: %"Z"u\n", ms->ms_entries);
+	printf("  Branch pages: %"Y"u\n", ms->ms_branch_pages);
+	printf("  Leaf pages: %"Y"u\n", ms->ms_leaf_pages);
+	printf("  Overflow pages: %"Y"u\n", ms->ms_overflow_pages);
+	printf("  Entries: %"Y"u\n", ms->ms_entries);
 }
 
 static void usage(char *prog)
@@ -125,11 +134,11 @@ int main(int argc, char *argv[])
 		(void)mdb_env_info(env, &mei);
 		printf("Environment Info\n");
 		printf("  Map address: %p\n", mei.me_mapaddr);
-		printf("  Map size: %"Z"u\n", mei.me_mapsize);
+		printf("  Map size: %"Y"u\n", mei.me_mapsize);
 		printf("  Page size: %u\n", mst.ms_psize);
-		printf("  Max pages: %"Z"u\n", mei.me_mapsize / mst.ms_psize);
-		printf("  Number of pages used: %"Z"u\n", mei.me_last_pgno+1);
-		printf("  Last transaction ID: %"Z"u\n", mei.me_last_txnid);
+		printf("  Max pages: %"Y"u\n", mei.me_mapsize / mst.ms_psize);
+		printf("  Number of pages used: %"Y"u\n", mei.me_last_pgno+1);
+		printf("  Last transaction ID: %"Y"u\n", mei.me_last_txnid);
 		printf("  Max readers: %u\n", mei.me_maxreaders);
 		printf("  Number of readers used: %u\n", mei.me_numreaders);
 	}
diff --git a/libraries/liblmdb/midl.h b/libraries/liblmdb/midl.h
index 2331e78398..8ca1653a1d 100644
--- a/libraries/liblmdb/midl.h
+++ b/libraries/liblmdb/midl.h
@@ -27,6 +27,7 @@
 #define _MDB_MIDL_H_
 
 #include <stddef.h>
+#include <inttypes.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -42,7 +43,11 @@ extern "C" {
 	/** A generic unsigned ID number. These were entryIDs in back-bdb.
 	 *	Preferably it should have the same size as a pointer.
 	 */
+#ifdef MDB_VL32
+typedef uint64_t MDB_ID;
+#else
 typedef size_t MDB_ID;
+#endif
 
 	/** An IDL is an ID List, a sorted array of IDs. The first
 	 * element of the array is a counter for how many actual
@@ -55,7 +60,11 @@ typedef MDB_ID *MDB_IDL;
 /* IDL sizes - likely should be even bigger
  *   limiting factors: sizeof(ID), thread stack size
  */
+#ifdef MDB_VL32
+#define	MDB_IDL_LOGN	10	/* DB_SIZE is 2^10, UM_SIZE is 2^11 */
+#else
 #define	MDB_IDL_LOGN	16	/* DB_SIZE is 2^16, UM_SIZE is 2^17 */
+#endif
 #define MDB_IDL_DB_SIZE		(1<<MDB_IDL_LOGN)
 #define MDB_IDL_UM_SIZE		(1<<(MDB_IDL_LOGN+1))
 

From 29ebd02efd72fa0b558b6d87fc32afc1061f97ff Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sat, 19 Dec 2015 23:58:39 +0000
Subject: [PATCH 192/504] MDB_VL32 support 64 bit DBs on 32 bit processors

---
 libraries/liblmdb/mdb.c  | 596 ++++++++++++++++++++++++++++++++++++++-
 libraries/liblmdb/midl.c |  62 ++++
 libraries/liblmdb/midl.h |  14 +
 3 files changed, 664 insertions(+), 8 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 0e190e9d13..c6a6802e3f 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1157,6 +1157,9 @@ struct MDB_txn {
 	/** Nested txn under this txn, set together with flag #MDB_TXN_HAS_CHILD */
 	MDB_txn		*mt_child;
 	pgno_t		mt_next_pgno;	/**< next unallocated page */
+#ifdef MDB_VL32
+	pgno_t		mt_last_pgno;	/**< last written page */
+#endif
 	/** The ID of this transaction. IDs are integers incrementing from 1.
 	 *	Only committed write transactions increment the ID. If a transaction
 	 *	aborts, the ID may be re-used by the next writer.
@@ -1203,6 +1206,19 @@ struct MDB_txn {
 	MDB_cursor	**mt_cursors;
 	/** Array of flags for each DB */
 	unsigned char	*mt_dbflags;
+#ifdef MDB_VL32
+	/** List of read-only pages (actually chunks) */
+	MDB_ID3L	mt_rpages;
+	/** We map chunks of 16 pages. Even though Windows uses 4KB pages, all
+	 * mappings must begin on 64KB boundaries. So we round off all pgnos to
+	 * a chunk boundary. We do the same on Linux for symmetry, and also to
+	 * reduce the frequency of mmap/munmap calls.
+	 */
+#define MDB_RPAGE_CHUNK	16
+#define MDB_TRPAGE_SIZE	4096	/**< size of #mt_rpages array of chunks */
+#define MDB_TRPAGE_MAX	(MDB_TRPAGE_SIZE-1)	/**< maximum chunk index */
+	unsigned int mt_rpcheck;	/**< threshold for reclaiming unref'd chunks */
+#endif
 	/**	Number of DB records in use, or 0 when the txn is finished.
 	 *	This number only ever increments until the txn finishes; we
 	 *	don't decrement it when individual DB handles are closed.
@@ -1315,6 +1331,9 @@ struct MDB_env {
 	HANDLE		me_fd;		/**< The main data file */
 	HANDLE		me_lfd;		/**< The lock file */
 	HANDLE		me_mfd;			/**< just for writing the meta pages */
+#if defined(MDB_VL32) && defined(_WIN32)
+	HANDLE		me_fmh;		/**< File Mapping handle */
+#endif
 	/** Failed to update the meta page. Probably an I/O error. */
 #define	MDB_FATAL_ERROR	0x80000000U
 	/** Some fields are initialized. */
@@ -1372,6 +1391,13 @@ struct MDB_env {
 #else
 	mdb_mutex_t	me_rmutex;
 	mdb_mutex_t	me_wmutex;
+#endif
+#ifdef MDB_VL32
+	MDB_ID3L	me_rpages;	/**< like #mt_rpages, but global to env */
+	mdb_mutex_t	me_rpmutex;	/**< control access to #me_rpages */
+#define MDB_ERPAGE_SIZE	16384
+#define MDB_ERPAGE_MAX	(MDB_ERPAGE_SIZE-1)
+	unsigned int me_rpcheck;
 #endif
 	void		*me_userctx;	 /**< User-settable context */
 	MDB_assert_func *me_assert_func; /**< Callback for assertion failures */
@@ -1894,6 +1920,41 @@ mdb_dlist_free(MDB_txn *txn)
 	dl[0].mid = 0;
 }
 
+#ifdef MDB_VL32
+static void
+mdb_page_unref(MDB_txn *txn, MDB_page *mp)
+{
+	pgno_t pgno;
+	MDB_ID3L tl = txn->mt_rpages;
+	unsigned x, rem;
+	if (mp->mp_flags & (P_SUBP|P_DIRTY))
+		return;
+	rem = mp->mp_pgno & (MDB_RPAGE_CHUNK-1);
+	pgno = mp->mp_pgno ^ rem;
+	x = mdb_mid3l_search(tl, pgno);
+	if (x != tl[0].mid && tl[x+1].mid == mp->mp_pgno)
+		x++;
+	if (tl[x].mref)
+		tl[x].mref--;
+}
+#define MDB_PAGE_UNREF(txn, mp)	mdb_page_unref(txn, mp)
+
+static void
+mdb_cursor_unref(MDB_cursor *mc)
+{
+	int i;
+	if (!mc->mc_pg[0] || IS_SUBP(mc->mc_pg[0]))
+		return;
+	for (i=0; i<mc->mc_snum; i++)
+		mdb_page_unref(mc->mc_txn, mc->mc_pg[i]);
+	mc->mc_snum = mc->mc_top = 0;
+	mc->mc_pg[0] = NULL;
+	mc->mc_flags &= ~C_INITIALIZED;
+}
+#else
+#define MDB_PAGE_UNREF(txn, mp)
+#endif /* MDB_VL32 */
+
 /** Loosen or free a single page.
  * Saves single pages to a list for future reuse
  * in this same txn. It has been pulled from the freeDB
@@ -2573,6 +2634,7 @@ done:
 			}
 		}
 	}
+	MDB_PAGE_UNREF(mc->mc_txn, mp);
 	return 0;
 
 fail:
@@ -2854,6 +2916,9 @@ mdb_txn_renew0(MDB_txn *txn)
 
 	/* Moved to here to avoid a data race in read TXNs */
 	txn->mt_next_pgno = meta->mm_last_pg+1;
+#ifdef MDB_VL32
+	txn->mt_last_pgno = txn->mt_next_pgno - 1;
+#endif
 
 	txn->mt_flags = flags;
 
@@ -2932,6 +2997,17 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 		DPRINTF(("calloc: %s", strerror(errno)));
 		return ENOMEM;
 	}
+#ifdef MDB_VL32
+	if (!parent) {
+		txn->mt_rpages = malloc(MDB_TRPAGE_SIZE * sizeof(MDB_ID3));
+		if (!txn->mt_rpages) {
+			free(txn);
+			return ENOMEM;
+		}
+		txn->mt_rpages[0].mid = 0;
+		txn->mt_rpcheck = MDB_TRPAGE_SIZE/2;
+	}
+#endif
 	txn->mt_dbxs = env->me_dbxs;	/* static */
 	txn->mt_dbs = (MDB_db *) ((char *)txn + tsize);
 	txn->mt_dbflags = (unsigned char *)txn + size - env->me_maxdbs;
@@ -2959,6 +3035,9 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 		parent->mt_child = txn;
 		txn->mt_parent = parent;
 		txn->mt_numdbs = parent->mt_numdbs;
+#ifdef MDB_VL32
+		txn->mt_rpages = parent->mt_rpages;
+#endif
 		memcpy(txn->mt_dbs, parent->mt_dbs, txn->mt_numdbs * sizeof(MDB_db));
 		/* Copy parent's mt_dbflags, but clear DB_NEW */
 		for (i=0; i<txn->mt_numdbs; i++)
@@ -2984,8 +3063,12 @@ renew:
 		rc = mdb_txn_renew0(txn);
 	}
 	if (rc) {
-		if (txn != env->me_txn0)
+		if (txn != env->me_txn0) {
+#ifdef MDB_VL32
+			free(txn->mt_rpages);
+#endif
 			free(txn);
+		}
 	} else {
 		txn->mt_flags |= flags;	/* could not change txn=me_txn0 earlier */
 		*ret = txn;
@@ -3110,7 +3193,31 @@ mdb_txn_end(MDB_txn *txn, unsigned mode)
 
 		mdb_midl_free(pghead);
 	}
-
+#ifdef MDB_VL32
+	if (!txn->mt_parent) {
+		MDB_ID3L el = env->me_rpages, tl = txn->mt_rpages;
+		unsigned i, x, n = tl[0].mid;
+		LOCK_MUTEX0(env->me_rpmutex);
+		for (i = 1; i <= n; i++) {
+			if (tl[i].mid & (MDB_RPAGE_CHUNK-1)) {
+				/* tmp overflow pages that we didn't share in env */
+				munmap(tl[i].mptr, tl[i].mcnt * env->me_psize);
+			} else {
+				x = mdb_mid3l_search(el, tl[i].mid);
+				if (tl[i].mptr == el[x].mptr) {
+					el[x].mref--;
+				} else {
+					/* another tmp overflow page */
+					munmap(tl[i].mptr, tl[i].mcnt * env->me_psize);
+				}
+			}
+		}
+		UNLOCK_MUTEX(env->me_rpmutex);
+		tl[0].mid = 0;
+		if (mode & MDB_END_FREE)
+			free(tl);
+	}
+#endif
 	if (mode & MDB_END_FREE)
 		free(txn);
 }
@@ -3460,6 +3567,10 @@ retry_seek:
 		n++;
 #endif	/* _WIN32 */
 	}
+#ifdef MDB_VL32
+	if (pgno > txn->mt_last_pgno)
+		txn->mt_last_pgno = pgno;
+#endif
 
 	/* MIPS has cache coherency issues, this is a no-op everywhere else
 	 * Note: for any size >= on-chip cache size, entire on-chip cache is
@@ -4046,11 +4157,27 @@ mdb_env_map(MDB_env *env, void *addr)
 	if (rc)
 		return rc;
 	map = addr;
+#ifdef MDB_VL32
+	msize = NUM_METAS * env->me_psize;
+#endif
 	rc = NtMapViewOfSection(mh, GetCurrentProcess(), &map, 0, 0, NULL, &msize, ViewUnmap, alloctype, pageprot);
+#ifdef MDB_VL32
+	env->me_fmh = mh;
+#else
 	NtClose(mh);
+#endif
 	if (rc)
 		return rc;
 	env->me_map = map;
+#else
+#ifdef MDB_VL32
+	(void) flags;
+	env->me_map = mmap(addr, NUM_METAS * env->me_psize, PROT_READ, MAP_SHARED,
+		env->me_fd, 0);
+	if (env->me_map == MAP_FAILED) {
+		env->me_map = NULL;
+		return ErrCode();
+	}
 #else
 	int prot = PROT_READ;
 	if (flags & MDB_WRITEMAP) {
@@ -4084,6 +4211,7 @@ mdb_env_map(MDB_env *env, void *addr)
 	 */
 	if (addr && env->me_map != addr)
 		return EBUSY;	/* TODO: Make a new MDB_* error code? */
+#endif
 
 	p = (MDB_page *)env->me_map;
 	env->me_metas[0] = METADATA(p);
@@ -4099,9 +4227,11 @@ mdb_env_set_mapsize(MDB_env *env, mdb_size_t size)
 	 * sure there are no active txns.
 	 */
 	if (env->me_map) {
-		int rc;
 		MDB_meta *meta;
+#ifndef MDB_VL32
 		void *old;
+		int rc;
+#endif
 		if (env->me_txn)
 			return EINVAL;
 		meta = mdb_env_pick_meta(env);
@@ -4113,12 +4243,17 @@ mdb_env_set_mapsize(MDB_env *env, mdb_size_t size)
 			if (size < minsize)
 				size = minsize;
 		}
+#ifndef MDB_VL32
+		/* For MDB_VL32 this bit is a noop since we dynamically remap
+		 * chunks of the DB anyway.
+		 */
 		munmap(env->me_map, env->me_mapsize);
 		env->me_mapsize = size;
 		old = (env->me_flags & MDB_FIXEDMAP) ? env->me_map : NULL;
 		rc = mdb_env_map(env, old);
 		if (rc)
 			return rc;
+#endif /* !MDB_VL32 */
 	}
 	env->me_mapsize = size;
 	if (env->me_psize)
@@ -4843,6 +4978,13 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 	env->me_wmutex->semnum = 1;
 	env->me_rmutex->locked = &env->me_txns->mti_rlocked;
 	env->me_wmutex->locked = &env->me_txns->mti_wlocked;
+#endif
+#ifdef MDB_VL32
+#ifdef _WIN32
+	env->me_rpmutex = CreateMutex(NULL, FALSE, NULL);
+#else
+	pthread_mutex_init(env->me_rpmutex, NULL);
+#endif
 #endif
 
 	return MDB_SUCCESS;
@@ -4883,6 +5025,17 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 	if (env->me_fd!=INVALID_HANDLE_VALUE || (flags & ~(CHANGEABLE|CHANGELESS)))
 		return EINVAL;
 
+#ifdef MDB_VL32
+	if (flags & MDB_WRITEMAP) {
+		/* silently ignore WRITEMAP in 32 bit mode */
+		flags ^= MDB_WRITEMAP;
+	}
+	if (flags & MDB_FIXEDMAP) {
+		/* cannot support FIXEDMAP */
+		return EINVAL;
+	}
+#endif
+
 	len = strlen(path);
 	if (flags & MDB_NOSUBDIR) {
 		rc = len + sizeof(LOCKSUFF) + len + 1;
@@ -4912,6 +5065,17 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 			  (env->me_dirty_list = calloc(MDB_IDL_UM_SIZE, sizeof(MDB_ID2)))))
 			rc = ENOMEM;
 	}
+#ifdef MDB_VL32
+	if (!rc) {
+		env->me_rpages = malloc(MDB_ERPAGE_SIZE * sizeof(MDB_ID3));
+		if (!env->me_rpages) {
+			rc = ENOMEM;
+			goto leave;
+		}
+		env->me_rpages[0].mid = 0;
+		env->me_rpcheck = MDB_ERPAGE_SIZE/2;
+	}
+#endif
 	env->me_flags = flags |= MDB_ENV_ACTIVE;
 	if (rc)
 		goto leave;
@@ -5010,6 +5174,16 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 				txn->mt_dbiseqs = (unsigned int *)(txn->mt_cursors + env->me_maxdbs);
 				txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs);
 				txn->mt_env = env;
+#ifdef MDB_VL32
+				txn->mt_rpages = malloc(MDB_TRPAGE_SIZE * sizeof(MDB_ID3));
+				if (!txn->mt_rpages) {
+					free(txn);
+					rc = ENOMEM;
+					goto leave;
+				}
+				txn->mt_rpages[0].mid = 0;
+				txn->mt_rpcheck = MDB_TRPAGE_SIZE/2;
+#endif
 				txn->mt_dbxs = env->me_dbxs;
 				txn->mt_flags = MDB_TXN_FINISHED;
 				env->me_txn0 = txn;
@@ -5049,6 +5223,13 @@ mdb_env_close0(MDB_env *env, int excl)
 	free(env->me_path);
 	free(env->me_dirty_list);
 	free(env->me_txn0);
+#ifdef MDB_VL32
+	{ unsigned int x;
+		for (x=1; x<=env->me_rpages[0].mid; x++)
+		munmap(env->me_rpages[x].mptr, env->me_rpages[x].mcnt * env->me_psize);
+	}
+	free(env->me_rpages);
+#endif
 	mdb_midl_free(env->me_free_pgs);
 
 	if (env->me_flags & MDB_ENV_TXKEY) {
@@ -5065,7 +5246,11 @@ mdb_env_close0(MDB_env *env, int excl)
 	}
 
 	if (env->me_map) {
+#ifdef MDB_VL32
+		munmap(env->me_map, NUM_METAS*env->me_psize);
+#else
 		munmap(env->me_map, env->me_mapsize);
+#endif
 	}
 	if (env->me_mfd != env->me_fd && env->me_mfd != INVALID_HANDLE_VALUE)
 		(void) close(env->me_mfd);
@@ -5130,6 +5315,13 @@ mdb_env_close0(MDB_env *env, int excl)
 #endif
 		(void) close(env->me_lfd);
 	}
+#ifdef MDB_VL32
+#ifdef _WIN32
+	if (env->me_rpmutex) CloseHandle(env->me_rpmutex);
+#else
+	pthread_mutex_destroy(env->me_rpmutex);
+#endif
+#endif
 
 	env->me_flags &= ~(MDB_ENV_ACTIVE|MDB_ENV_TXKEY);
 }
@@ -5396,6 +5588,309 @@ mdb_cursor_push(MDB_cursor *mc, MDB_page *mp)
 	return MDB_SUCCESS;
 }
 
+#ifdef MDB_VL32
+/** Map a read-only page.
+ * There are two levels of tracking in use, a per-txn list and a per-env list.
+ * ref'ing and unref'ing the per-txn list is faster since it requires no
+ * locking. Pages are cached in the per-env list for global reuse, and a lock
+ * is required. Pages are not immediately unmapped when their refcnt goes to
+ * zero; they hang around in case they will be reused again soon.
+ *
+ * When the per-txn list gets full, all pages with refcnt=0 are purged from the
+ * list and their refcnts in the per-env list are decremented.
+ *
+ * When the per-env list gets full, all pages with refcnt=0 are purged from the
+ * list and their pages are unmapped.
+ *
+ * @note "full" means the list has reached its respective rpcheck threshold.
+ * This threshold slowly raises if no pages could be purged on a given check,
+ * and returns to its original value when enough pages were purged.
+ *
+ * If purging doesn't free any slots, filling the per-txn list will return
+ * MDB_TXN_FULL, and filling the per-env list returns MDB_MAP_FULL.
+ *
+ * Reference tracking in a txn is imperfect, pages can linger with non-zero
+ * refcnt even without active references. It was deemed to be too invasive
+ * to add unrefs in every required location. However, all pages are unref'd
+ * at the end of the transaction. This guarantees that no stale references
+ * linger in the per-env list.
+ *
+ * Usually we map chunks of 16 pages at a time, but if an overflow page begins
+ * at the tail of the chunk we extend the chunk to include the entire overflow
+ * page. Unfortunately, pages can be turned into overflow pages after their
+ * chunk was already mapped. In that case we must remap the chunk if the
+ * overflow page is referenced. If the chunk's refcnt is 0 we can just remap
+ * it, otherwise we temporarily map a new chunk just for the overflow page.
+ *
+ * @note this chunk handling means we cannot guarantee that a data item
+ * returned from the DB will stay alive for the duration of the transaction:
+ *   We unref pages as soon as a cursor moves away from the page
+ *   A subsequent op may cause a purge, which may unmap any unref'd chunks
+ * The caller must copy the data if it must be used later in the same txn.
+ *
+ * Also - our reference counting revolves around cursors, but overflow pages
+ * aren't ever pointed to by a cursor. As such, they always remain referenced
+ * in a txn until it ends.
+ *
+ * @param[in] txn the transaction for this access.
+ * @param[in] pgno the page number for the page to retrieve.
+ * @param[out] ret address of a pointer where the page's address will be stored.
+ * @return 0 on success, non-zero on failure.
+ */
+static int
+mdb_rpage_get(MDB_txn *txn, pgno_t pg0, MDB_page **ret)
+{
+	MDB_env *env = txn->mt_env;
+	MDB_page *p;
+	MDB_ID3L tl = txn->mt_rpages;
+	MDB_ID3L el = env->me_rpages;
+	MDB_ID3 id3;
+	unsigned x, rem;
+	pgno_t pgno;
+	int rc, retries = 1;
+#ifdef _WIN32
+	LARGE_INTEGER off;
+	SIZE_T len;
+#define SET_OFF(off,val)	off.QuadPart = val
+#define MAP(rc,env,addr,len,off)	\
+	addr = NULL; \
+	rc = NtMapViewOfSection(env->me_fmh, GetCurrentProcess(), &addr, 0, \
+		len, &off, &len, ViewUnmap, MEM_RESERVE, PAGE_READONLY)
+#else
+	off_t off;
+	size_t len;
+#define SET_OFF(off,val)	off = val
+#define MAP(rc,env,addr,len,off)	\
+	addr = mmap(NULL, len, PROT_READ, MAP_SHARED, env->me_fd, off); \
+	rc = (addr == MAP_FAILED) ? errno : 0
+#endif
+
+	/* remember the offset of the actual page number, so we can
+	 * return the correct pointer at the end.
+	 */
+	rem = pg0 & (MDB_RPAGE_CHUNK-1);
+	pgno = pg0 ^ rem;
+
+	id3.mid = 0;
+	x = mdb_mid3l_search(tl, pgno);
+	if (x <= tl[0].mid && tl[x].mid == pgno) {
+		if (x != tl[0].mid && tl[x+1].mid == pg0)
+			x++;
+		/* check for overflow size */
+		p = (MDB_page *)((char *)tl[x].mptr + rem * env->me_psize);
+		if (IS_OVERFLOW(p) && p->mp_pages + rem > tl[x].mcnt) {
+			id3.mcnt = p->mp_pages + rem;
+			len = id3.mcnt * env->me_psize;
+			SET_OFF(off, pgno * env->me_psize);
+			MAP(rc, env, id3.mptr, len, off);
+			if (rc)
+				return rc;
+			/* check for local-only page */
+			if (rem) {
+				mdb_tassert(txn, tl[x].mid != pg0);
+				/* hope there's room to insert this locally.
+				 * setting mid here tells later code to just insert
+				 * this id3 instead of searching for a match.
+				 */
+				id3.mid = pg0;
+				goto notlocal;
+			} else {
+				/* ignore the mapping we got from env, use new one */
+				tl[x].mptr = id3.mptr;
+				tl[x].mcnt = id3.mcnt;
+				/* if no active ref, see if we can replace in env */
+				if (!tl[x].mref) {
+					unsigned i;
+					LOCK_MUTEX0(env->me_rpmutex);
+					i = mdb_mid3l_search(el, tl[x].mid);
+					if (el[i].mref == 1) {
+						/* just us, replace it */
+						munmap(el[i].mptr, el[i].mcnt * env->me_psize);
+						el[i].mptr = tl[x].mptr;
+						el[i].mcnt = tl[x].mcnt;
+					} else {
+						/* there are others, remove ourself */
+						el[i].mref--;
+					}
+					UNLOCK_MUTEX(env->me_rpmutex);
+				}
+			}
+		}
+		id3.mptr = tl[x].mptr;
+		id3.mcnt = tl[x].mcnt;
+		tl[x].mref++;
+		goto ok;
+	}
+
+notlocal:
+	if (tl[0].mid >= MDB_TRPAGE_MAX - txn->mt_rpcheck) {
+		unsigned i, y;
+		/* purge unref'd pages from our list and unref in env */
+		LOCK_MUTEX0(env->me_rpmutex);
+retry:
+		y = 0;
+		for (i=1; i<tl[0].mid; i++) {
+			if (!tl[i].mref) {
+				if (!y) y = i;
+				/* tmp overflow pages don't go to env */
+				if (tl[i].mid & (MDB_RPAGE_CHUNK-1)) {
+					munmap(tl[i].mptr, tl[i].mcnt * env->me_psize);
+					continue;
+				}
+				x = mdb_mid3l_search(el, tl[i].mid);
+				el[x].mref--;
+			}
+		}
+		UNLOCK_MUTEX(env->me_rpmutex);
+		if (!y) {
+			/* we didn't find any unref'd chunks.
+			 * if we're out of room, fail.
+			 */
+			if (tl[0].mid >= MDB_TRPAGE_MAX)
+				return MDB_TXN_FULL;
+			/* otherwise, raise threshold for next time around
+			 * and let this go.
+			 */
+			txn->mt_rpcheck /= 2;
+		} else {
+			/* we found some unused; consolidate the list */
+			for (i=y+1; i<= tl[0].mid; i++)
+				if (tl[i].mref)
+					tl[y++] = tl[i];
+			tl[0].mid = y-1;
+			/* decrease the check threshold toward its original value */
+			if (!txn->mt_rpcheck)
+				txn->mt_rpcheck = 1;
+			while (txn->mt_rpcheck < tl[0].mid && txn->mt_rpcheck < MDB_TRPAGE_SIZE/2)
+				txn->mt_rpcheck *= 2;
+		}
+	}
+	if (tl[0].mid < MDB_TRPAGE_SIZE) {
+		id3.mref = 1;
+		if (id3.mid)
+			goto found;
+		len = env->me_psize * MDB_RPAGE_CHUNK;
+		id3.mid = pgno;
+		id3.mcnt = MDB_RPAGE_CHUNK;
+
+		/* search for page in env */
+		LOCK_MUTEX0(env->me_rpmutex);
+		x = mdb_mid3l_search(el, pgno);
+		if (x <= el[0].mid && el[x].mid == pgno) {
+			id3.mptr = el[x].mptr;
+			id3.mcnt = el[x].mcnt;
+			/* check for overflow size */
+			p = (MDB_page *)((char *)id3.mptr + rem * env->me_psize);
+			if (IS_OVERFLOW(p) && p->mp_pages + rem > id3.mcnt) {
+				id3.mcnt = p->mp_pages + rem;
+				len = id3.mcnt * env->me_psize;
+				SET_OFF(off, pgno * env->me_psize);
+				MAP(rc, env, id3.mptr, len, off);
+				if (rc)
+					goto fail;
+				if (!el[x].mref) {
+					munmap(el[x].mptr, el[x].mcnt);
+					el[x].mptr = id3.mptr;
+					el[x].mcnt = id3.mcnt;
+				} else {
+					id3.mid = pg0;
+					UNLOCK_MUTEX(env->me_rpmutex);
+					goto found;
+				}
+			}
+			el[x].mref++;
+			UNLOCK_MUTEX(env->me_rpmutex);
+			goto found;
+		}
+		if (el[0].mid >= MDB_ERPAGE_MAX - env->me_rpcheck) {
+			/* purge unref'd pages */
+			unsigned i, y = 0;
+			for (i=1; i<el[0].mid; i++) {
+				if (!el[i].mref) {
+					if (!y) y = i;
+					munmap(el[i].mptr, env->me_psize * el[i].mcnt);
+				}
+			}
+			if (!y) {
+				if (retries) {
+					/* see if we can unref some local pages */
+					retries--;
+					id3.mid = 0;
+					goto retry;
+				}
+				if (el[0].mid >= MDB_ERPAGE_MAX) {
+					UNLOCK_MUTEX(env->me_rpmutex);
+					return MDB_MAP_FULL;
+				}
+				env->me_rpcheck /= 2;
+			} else {
+				for (i=y+1; i<= el[0].mid; i++)
+					if (el[i].mref)
+						el[y++] = el[i];
+				el[0].mid = y-1;
+				if (!env->me_rpcheck)
+					env->me_rpcheck = 1;
+				while (env->me_rpcheck < el[0].mid && env->me_rpcheck < MDB_ERPAGE_SIZE/2)
+					env->me_rpcheck *= 2;
+			}
+		}
+		SET_OFF(off, pgno * env->me_psize);
+		MAP(rc, env, id3.mptr, len, off);
+		if (rc) {
+fail:
+			UNLOCK_MUTEX(env->me_rpmutex);
+			return rc;
+		}
+		/* If this page is far enough from the end of the env, scan for
+		 * any overflow pages that would spill onto another block.
+		 * Note we must compare against mt_last_pgno, the last written
+		 * page in the environment. Not mt_next_pgno, which increases
+		 * for every newly allocated (but not yet written) page. If
+		 * we scanned beyond the last written page we'd get a bus error.
+		 */
+		if (pgno + MDB_RPAGE_CHUNK <= txn->mt_last_pgno) {
+			int i;
+			char *cp = (char *)id3.mptr + rem * env->me_psize;
+			for (i=rem; i<MDB_RPAGE_CHUNK;) {
+				p = (MDB_page *)cp;
+				if (IS_OVERFLOW(p)) {
+					int nop = p->mp_pages;
+					if (nop + i > MDB_RPAGE_CHUNK) {
+						munmap(id3.mptr, len);
+						id3.mcnt = nop + i;
+						len = id3.mcnt * env->me_psize;
+						MAP(rc, env, id3.mptr, len, off);
+						if (rc)
+							goto fail;
+						break;
+					}
+					i += nop;
+					cp += nop * env->me_psize;
+				} else {
+					i++;
+					cp += env->me_psize;
+				}
+			}
+		}
+		mdb_mid3l_insert(el, &id3);
+		UNLOCK_MUTEX(env->me_rpmutex);
+found:
+		mdb_mid3l_insert(tl, &id3);
+	} else {
+		return MDB_TXN_FULL;
+	}
+ok:
+	p = (MDB_page *)((char *)id3.mptr + rem * env->me_psize);
+#if MDB_DEBUG	/* we don't need this check any more */
+	if (IS_OVERFLOW(p)) {
+		mdb_tassert(txn, p->mp_pages + rem <= id3.mcnt);
+	}
+#endif
+	*ret = p;
+	return MDB_SUCCESS;
+}
+#endif
+
 /** Find the address of the page corresponding to a given page number.
  * @param[in] txn the transaction for this access.
  * @param[in] pgno the page number for the page to retrieve.
@@ -5406,7 +5901,9 @@ mdb_cursor_push(MDB_cursor *mc, MDB_page *mp)
 static int
 mdb_page_get(MDB_txn *txn, pgno_t pgno, MDB_page **ret, int *lvl)
 {
+#ifndef MDB_VL32
 	MDB_env *env = txn->mt_env;
+#endif
 	MDB_page *p = NULL;
 	int level;
 
@@ -5425,7 +5922,13 @@ mdb_page_get(MDB_txn *txn, pgno_t pgno, MDB_page **ret, int *lvl)
 				MDB_ID pn = pgno << 1;
 				x = mdb_midl_search(tx2->mt_spill_pgs, pn);
 				if (x <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[x] == pn) {
+#ifdef MDB_VL32
+					int rc = mdb_rpage_get(txn, pgno, &p);
+					if (rc)
+						return rc;
+#else
 					p = (MDB_page *)(env->me_map + env->me_psize * pgno);
+#endif
 					goto done;
 				}
 			}
@@ -5442,7 +5945,15 @@ mdb_page_get(MDB_txn *txn, pgno_t pgno, MDB_page **ret, int *lvl)
 
 	if (pgno < txn->mt_next_pgno) {
 		level = 0;
+#ifdef MDB_VL32
+		{
+			int rc = mdb_rpage_get(txn, pgno, &p);
+			if (rc)
+				return rc;
+		}
+#else
 		p = (MDB_page *)(env->me_map + env->me_psize * pgno);
+#endif
 	} else {
 		DPRINTF(("page %"Y"u not found", pgno));
 		txn->mt_flags |= MDB_TXN_ERROR;
@@ -5617,10 +6128,22 @@ mdb_page_search(MDB_cursor *mc, MDB_val *key, int flags)
 	}
 
 	mdb_cassert(mc, root > 1);
-	if (!mc->mc_pg[0] || mc->mc_pg[0]->mp_pgno != root)
+	if (!mc->mc_pg[0] || mc->mc_pg[0]->mp_pgno != root) {
+#ifdef MDB_VL32
+		if (mc->mc_pg[0])
+			MDB_PAGE_UNREF(mc->mc_txn, mc->mc_pg[0]);
+#endif
 		if ((rc = mdb_page_get(mc->mc_txn, root, &mc->mc_pg[0], NULL)) != 0)
 			return rc;
+	}
 
+#ifdef MDB_VL32
+	{
+		int i;
+		for (i=1; i<mc->mc_snum; i++)
+			MDB_PAGE_UNREF(mc->mc_txn, mc->mc_pg[i]);
+	}
+#endif
 	mc->mc_snum = 1;
 	mc->mc_top = 0;
 
@@ -5752,7 +6275,7 @@ mdb_get(MDB_txn *txn, MDB_dbi dbi,
 {
 	MDB_cursor	mc;
 	MDB_xcursor	mx;
-	int exact = 0;
+	int exact = 0, rc;
 	DKBUF;
 
 	DPRINTF(("===> get db %u key [%s]", dbi, DKEY(key)));
@@ -5764,7 +6287,16 @@ mdb_get(MDB_txn *txn, MDB_dbi dbi,
 		return MDB_BAD_TXN;
 
 	mdb_cursor_init(&mc, txn, dbi, &mx);
-	return mdb_cursor_set(&mc, key, data, MDB_SET, &exact);
+	rc = mdb_cursor_set(&mc, key, data, MDB_SET, &exact);
+#ifdef MDB_VL32
+	{
+		/* unref all the pages - caller must copy the data
+		 * before doing anything else
+		 */
+		mdb_cursor_unref(&mc);
+	}
+#endif
+	return rc;
 }
 
 /** Find a sibling for a page.
@@ -5781,11 +6313,17 @@ mdb_cursor_sibling(MDB_cursor *mc, int move_right)
 	int		 rc;
 	MDB_node	*indx;
 	MDB_page	*mp;
+#ifdef MDB_VL32
+	MDB_page	*op;
+#endif
 
 	if (mc->mc_snum < 2) {
 		return MDB_NOTFOUND;		/* root has no siblings */
 	}
 
+#ifdef MDB_VL32
+	op = mc->mc_pg[mc->mc_top];
+#endif
 	mdb_cursor_pop(mc);
 	DPRINTF(("parent page is page %"Y"u, index %u",
 		mc->mc_pg[mc->mc_top]->mp_pgno, mc->mc_ki[mc->mc_top]));
@@ -5810,6 +6348,8 @@ mdb_cursor_sibling(MDB_cursor *mc, int move_right)
 	}
 	mdb_cassert(mc, IS_BRANCH(mc->mc_pg[mc->mc_top]));
 
+	MDB_PAGE_UNREF(mc->mc_txn, op);
+
 	indx = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
 	if ((rc = mdb_page_get(mc->mc_txn, NODEPGNO(indx), &mp, NULL)) != 0) {
 		/* mc will be inconsistent if caller does mc_snum++ as above */
@@ -5851,6 +6391,13 @@ mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
 					return rc;
 				}
 			}
+#ifdef MDB_VL32
+			else {
+				if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) {
+					mdb_cursor_unref(&mc->mc_xcursor->mx_cursor);
+				}
+			}
+#endif
 		} else {
 			mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF);
 			if (op == MDB_NEXT_DUP)
@@ -5930,6 +6477,13 @@ mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
 					return rc;
 				}
 			}
+#ifdef MDB_VL32
+			else {
+				if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) {
+					mdb_cursor_unref(&mc->mc_xcursor->mx_cursor);
+				}
+			}
+#endif
 		} else {
 			mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF);
 			if (op == MDB_PREV_DUP)
@@ -6129,6 +6683,11 @@ set1:
 		return MDB_SUCCESS;
 	}
 
+#ifdef MDB_VL32
+	if (mc->mc_xcursor && mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) {
+		mdb_cursor_unref(&mc->mc_xcursor->mx_cursor);
+	}
+#endif
 	if (F_ISSET(leaf->mn_flags, F_DUPDATA)) {
 		mdb_xcursor_init1(mc, leaf);
 	}
@@ -6189,8 +6748,14 @@ mdb_cursor_first(MDB_cursor *mc, MDB_val *key, MDB_val *data)
 	int		 rc;
 	MDB_node	*leaf;
 
-	if (mc->mc_xcursor)
+	if (mc->mc_xcursor) {
+#ifdef MDB_VL32
+		if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) {
+			mdb_cursor_unref(&mc->mc_xcursor->mx_cursor);
+		}
+#endif
 		mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF);
+	}
 
 	if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) {
 		rc = mdb_page_search(mc, NULL, MDB_PS_FIRST);
@@ -6233,8 +6798,14 @@ mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data)
 	int		 rc;
 	MDB_node	*leaf;
 
-	if (mc->mc_xcursor)
+	if (mc->mc_xcursor) {
+#ifdef MDB_VL32
+		if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) {
+			mdb_cursor_unref(&mc->mc_xcursor->mx_cursor);
+		}
+#endif
 		mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF);
+	}
 
 	if (!(mc->mc_flags & C_EOF)) {
 
@@ -9854,6 +10425,11 @@ mdb_drop0(MDB_cursor *mc, int subs)
 			mdb_cursor_pop(mc);
 
 		mdb_cursor_copy(mc, &mx);
+#ifdef MDB_VL32
+		/* bump refcount for mx's pages */
+		for (i=0; i<mc->mc_snum; i++)
+			mdb_page_get(txn, mc->mc_pg[i]->mp_pgno, &mx.mc_pg[i], NULL);
+#endif
 		while (mc->mc_snum > 0) {
 			MDB_page *mp = mc->mc_pg[mc->mc_top];
 			unsigned n = NUMKEYS(mp);
@@ -9913,6 +10489,10 @@ mdb_drop0(MDB_cursor *mc, int subs)
 done:
 		if (rc)
 			txn->mt_flags |= MDB_TXN_ERROR;
+#ifdef MDB_VL32
+		/* drop refcount for mx's pages */
+		mdb_cursor_unref(&mx);
+#endif
 	} else if (rc == MDB_NOTFOUND) {
 		rc = MDB_SUCCESS;
 	}
diff --git a/libraries/liblmdb/midl.c b/libraries/liblmdb/midl.c
index 57a9d4920e..e2005d16cf 100644
--- a/libraries/liblmdb/midl.c
+++ b/libraries/liblmdb/midl.c
@@ -354,5 +354,67 @@ int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id )
 	return 0;
 }
 
+#ifdef MDB_VL32
+unsigned mdb_mid3l_search( MDB_ID3L ids, MDB_ID id )
+{
+	/*
+	 * binary search of id in ids
+	 * if found, returns position of id
+	 * if not found, returns first position greater than id
+	 */
+	unsigned base = 0;
+	unsigned cursor = 1;
+	int val = 0;
+	unsigned n = (unsigned)ids[0].mid;
+
+	while( 0 < n ) {
+		unsigned pivot = n >> 1;
+		cursor = base + pivot + 1;
+		val = CMP( id, ids[cursor].mid );
+
+		if( val < 0 ) {
+			n = pivot;
+
+		} else if ( val > 0 ) {
+			base = cursor;
+			n -= pivot + 1;
+
+		} else {
+			return cursor;
+		}
+	}
+
+	if( val > 0 ) {
+		++cursor;
+	}
+	return cursor;
+}
+
+int mdb_mid3l_insert( MDB_ID3L ids, MDB_ID3 *id )
+{
+	unsigned x, i;
+
+	x = mdb_mid3l_search( ids, id->mid );
+
+	if( x < 1 ) {
+		/* internal error */
+		return -2;
+	}
+
+	if ( x <= ids[0].mid && ids[x].mid == id->mid ) {
+		/* duplicate */
+		return -1;
+	}
+
+	/* insert id */
+	ids[0].mid++;
+	for (i=(unsigned)ids[0].mid; i>x; i--)
+		ids[i] = ids[i-1];
+	ids[x] = *id;
+
+	return 0;
+}
+#endif /* MDB_VL32 */
+
 /** @} */
 /** @} */
diff --git a/libraries/liblmdb/midl.h b/libraries/liblmdb/midl.h
index 8ca1653a1d..ed1d75e36b 100644
--- a/libraries/liblmdb/midl.h
+++ b/libraries/liblmdb/midl.h
@@ -186,6 +186,20 @@ int mdb_mid2l_insert( MDB_ID2L ids, MDB_ID2 *id );
 	 */
 int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id );
 
+#ifdef MDB_VL32
+typedef struct MDB_ID3 {
+	MDB_ID mid;		/**< The ID */
+	void *mptr;		/**< The pointer */
+	unsigned int mcnt;		/**< Number of pages */
+	unsigned int mref;		/**< Refcounter */
+} MDB_ID3;
+
+typedef MDB_ID3 *MDB_ID3L;
+
+unsigned mdb_mid3l_search( MDB_ID3L ids, MDB_ID id );
+int mdb_mid3l_insert( MDB_ID3L ids, MDB_ID3 *id );
+
+#endif /* MDB_VL32 */
 /** @} */
 /** @} */
 #ifdef __cplusplus

From 860527f66e461bd34c53e48e8525e11f6652a8b7 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sun, 20 Dec 2015 00:22:13 +0000
Subject: [PATCH 193/504] Note that we're prepping for 1.0

---
 libraries/liblmdb/lmdb.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index d2bf595058..ff19c0afe8 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -202,7 +202,7 @@ typedef int mdb_filehandle_t;
 /** Library minor version */
 #define MDB_VERSION_MINOR	9
 /** Library patch version */
-#define MDB_VERSION_PATCH	16
+#define MDB_VERSION_PATCH	70
 
 /** Combine args a,b,c into a single integer for easy version comparisons */
 #define MDB_VERINT(a,b,c)	(((a) << 24) | ((b) << 16) | (c))
@@ -212,7 +212,7 @@ typedef int mdb_filehandle_t;
 	MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH)
 
 /** The release date of this library version */
-#define MDB_VERSION_DATE	"August 14, 2015"
+#define MDB_VERSION_DATE	"December 19, 2015"
 
 /** A stringifier for the version info */
 #define MDB_VERSTR(a,b,c,d)	"LMDB " #a "." #b "." #c ": (" d ")"

From 209b56fead1afe8273db6c714c0a74a9c09b9cf6 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 21 Dec 2015 02:36:20 +0000
Subject: [PATCH 194/504] ITS#8324 fix for WRITEMAP

We called FlushViewOfFile with (map,mapsize) which worked fine
when we had allocated the entire map already. Now we have to make
sure to only flush as much as was actually written. Add a numpgs
argument to tell how much to flush in env_sync0().
---
 libraries/liblmdb/mdb.c | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index c6a6802e3f..a02ce5c108 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2643,7 +2643,7 @@ fail:
 }
 
 int
-mdb_env_sync(MDB_env *env, int force)
+mdb_env_sync0(MDB_env *env, int force, pgno_t numpgs)
 {
 	int rc = 0;
 	if (env->me_flags & MDB_RDONLY)
@@ -2652,7 +2652,7 @@ mdb_env_sync(MDB_env *env, int force)
 		if (env->me_flags & MDB_WRITEMAP) {
 			int flags = ((env->me_flags & MDB_MAPASYNC) && !force)
 				? MS_ASYNC : MS_SYNC;
-			if (MDB_MSYNC(env->me_map, env->me_mapsize, flags))
+			if (MDB_MSYNC(env->me_map, env->me_psize * numpgs, flags))
 				rc = ErrCode();
 #ifdef _WIN32
 			else if (flags == MS_SYNC && MDB_FDATASYNC(env->me_fd))
@@ -2672,6 +2672,13 @@ mdb_env_sync(MDB_env *env, int force)
 	return rc;
 }
 
+int
+mdb_env_sync(MDB_env *env, int force)
+{
+	MDB_meta *m = mdb_env_pick_meta(env);
+	return mdb_env_sync0(env, force, m->mm_last_pg+1);
+}
+
 /** Back up parent txn's cursors, then grab the originals for tracking */
 static int
 mdb_cursor_shadow(MDB_txn *src, MDB_txn *dst)
@@ -3813,7 +3820,7 @@ mdb_txn_commit(MDB_txn *txn)
 	if ((rc = mdb_page_flush(txn, 0)))
 		goto fail;
 	if (!F_ISSET(txn->mt_flags, MDB_TXN_NOSYNC) &&
-		(rc = mdb_env_sync(env, 0)))
+		(rc = mdb_env_sync0(env, 0, txn->mt_next_pgno)))
 		goto fail;
 	if ((rc = mdb_env_write_meta(txn)))
 		goto fail;

From 397d85d13c4f0c75cbe32768094090aa90bd0925 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sun, 23 Aug 2015 20:33:02 +0200
Subject: [PATCH 195/504] Pass cursor to mdb_page_get(), mdb_node_read().

No change in behavior.
---
 libraries/liblmdb/mdb.c | 70 ++++++++++++++++++++---------------------
 1 file changed, 35 insertions(+), 35 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index a02ce5c108..71e20a3c59 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1444,7 +1444,7 @@ enum {
 #define MDB_END_SLOT MDB_NOTLS	/**< release any reader slot if #MDB_NOTLS */
 static void mdb_txn_end(MDB_txn *txn, unsigned mode);
 
-static int  mdb_page_get(MDB_txn *txn, pgno_t pgno, MDB_page **mp, int *lvl);
+static int  mdb_page_get(MDB_cursor *mc, pgno_t pgno, MDB_page **mp, int *lvl);
 static int  mdb_page_search_root(MDB_cursor *mc,
 			    MDB_val *key, int modify);
 #define MDB_PS_MODIFY	1
@@ -1473,7 +1473,7 @@ static int  mdb_node_add(MDB_cursor *mc, indx_t indx,
 static void mdb_node_del(MDB_cursor *mc, int ksize);
 static void mdb_node_shrink(MDB_page *mp, indx_t indx);
 static int	mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft);
-static int  mdb_node_read(MDB_txn *txn, MDB_node *leaf, MDB_val *data);
+static int  mdb_node_read(MDB_cursor *mc, MDB_node *leaf, MDB_val *data);
 static size_t	mdb_leaf_size(MDB_env *env, MDB_val *key, MDB_val *data);
 static size_t	mdb_branch_size(MDB_env *env, MDB_val *key);
 
@@ -2023,7 +2023,7 @@ mdb_pages_xkeep(MDB_cursor *mc, unsigned pflags, int all)
 {
 	enum { Mask = P_SUBP|P_DIRTY|P_LOOSE|P_KEEP };
 	MDB_txn *txn = mc->mc_txn;
-	MDB_cursor *m3;
+	MDB_cursor *m3, *m0 = mc;
 	MDB_xcursor *mx;
 	MDB_page *dp, *mp;
 	MDB_node *leaf;
@@ -2066,7 +2066,7 @@ mdb_pages_xkeep(MDB_cursor *mc, unsigned pflags, int all)
 				pgno_t pgno = txn->mt_dbs[i].md_root;
 				if (pgno == P_INVALID)
 					continue;
-				if ((rc = mdb_page_get(txn, pgno, &dp, &level)) != MDB_SUCCESS)
+				if ((rc = mdb_page_get(m0, pgno, &dp, &level)) != MDB_SUCCESS)
 					break;
 				if ((dp->mp_flags & Mask) == pflags && level <= 1)
 					dp->mp_flags ^= P_KEEP;
@@ -2370,7 +2370,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
 		}
 		np = m2.mc_pg[m2.mc_top];
 		leaf = NODEPTR(np, m2.mc_ki[m2.mc_top]);
-		if ((rc = mdb_node_read(txn, leaf, &data)) != MDB_SUCCESS)
+		if ((rc = mdb_node_read(&m2, leaf, &data)) != MDB_SUCCESS)
 			return rc;
 
 		idl = (MDB_ID *) data.mv_data;
@@ -5899,15 +5899,16 @@ ok:
 #endif
 
 /** Find the address of the page corresponding to a given page number.
- * @param[in] txn the transaction for this access.
+ * @param[in] mc the cursor accessing the page.
  * @param[in] pgno the page number for the page to retrieve.
  * @param[out] ret address of a pointer where the page's address will be stored.
  * @param[out] lvl dirty_list inheritance level of found page. 1=current txn, 0=mapped page.
  * @return 0 on success, non-zero on failure.
  */
 static int
-mdb_page_get(MDB_txn *txn, pgno_t pgno, MDB_page **ret, int *lvl)
+mdb_page_get(MDB_cursor *mc, pgno_t pgno, MDB_page **ret, int *lvl)
 {
+	MDB_txn *txn = mc->mc_txn;
 #ifndef MDB_VL32
 	MDB_env *env = txn->mt_env;
 #endif
@@ -6018,7 +6019,7 @@ mdb_page_search_root(MDB_cursor *mc, MDB_val *key, int flags)
 		mdb_cassert(mc, i < NUMKEYS(mp));
 		node = NODEPTR(mp, i);
 
-		if ((rc = mdb_page_get(mc->mc_txn, NODEPGNO(node), &mp, NULL)) != 0)
+		if ((rc = mdb_page_get(mc, NODEPGNO(node), &mp, NULL)) != 0)
 			return rc;
 
 		mc->mc_ki[mc->mc_top] = i;
@@ -6060,7 +6061,7 @@ mdb_page_search_lowest(MDB_cursor *mc)
 	MDB_node	*node = NODEPTR(mp, 0);
 	int rc;
 
-	if ((rc = mdb_page_get(mc->mc_txn, NODEPGNO(node), &mp, NULL)) != 0)
+	if ((rc = mdb_page_get(mc, NODEPGNO(node), &mp, NULL)) != 0)
 		return rc;
 
 	mc->mc_ki[mc->mc_top] = 0;
@@ -6112,7 +6113,7 @@ mdb_page_search(MDB_cursor *mc, MDB_val *key, int flags)
 						return MDB_NOTFOUND;
 					if ((leaf->mn_flags & (F_DUPDATA|F_SUBDATA)) != F_SUBDATA)
 						return MDB_INCOMPATIBLE; /* not a named DB */
-					rc = mdb_node_read(mc->mc_txn, leaf, &data);
+					rc = mdb_node_read(&mc2, leaf, &data);
 					if (rc)
 						return rc;
 					memcpy(&flags, ((char *) data.mv_data + offsetof(MDB_db, md_flags)),
@@ -6140,7 +6141,7 @@ mdb_page_search(MDB_cursor *mc, MDB_val *key, int flags)
 		if (mc->mc_pg[0])
 			MDB_PAGE_UNREF(mc->mc_txn, mc->mc_pg[0]);
 #endif
-		if ((rc = mdb_page_get(mc->mc_txn, root, &mc->mc_pg[0], NULL)) != 0)
+		if ((rc = mdb_page_get(mc, root, &mc->mc_pg[0], NULL)) != 0)
 			return rc;
 	}
 
@@ -6245,13 +6246,13 @@ release:
 }
 
 /** Return the data associated with a given node.
- * @param[in] txn The transaction for this operation.
+ * @param[in] mc The cursor for this operation.
  * @param[in] leaf The node being read.
  * @param[out] data Updated to point to the node's data.
  * @return 0 on success, non-zero on failure.
  */
 static int
-mdb_node_read(MDB_txn *txn, MDB_node *leaf, MDB_val *data)
+mdb_node_read(MDB_cursor *mc, MDB_node *leaf, MDB_val *data)
 {
 	MDB_page	*omp;		/* overflow page */
 	pgno_t		 pgno;
@@ -6267,7 +6268,7 @@ mdb_node_read(MDB_txn *txn, MDB_node *leaf, MDB_val *data)
 	 */
 	data->mv_size = NODEDSZ(leaf);
 	memcpy(&pgno, NODEDATA(leaf), sizeof(pgno));
-	if ((rc = mdb_page_get(txn, pgno, &omp, NULL)) != 0) {
+	if ((rc = mdb_page_get(mc, pgno, &omp, NULL)) != 0) {
 		DPRINTF(("read overflow page %"Y"u failed", pgno));
 		return rc;
 	}
@@ -6358,7 +6359,7 @@ mdb_cursor_sibling(MDB_cursor *mc, int move_right)
 	MDB_PAGE_UNREF(mc->mc_txn, op);
 
 	indx = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
-	if ((rc = mdb_page_get(mc->mc_txn, NODEPGNO(indx), &mp, NULL)) != 0) {
+	if ((rc = mdb_page_get(mc, NODEPGNO(indx), &mp, NULL)) != 0) {
 		/* mc will be inconsistent if caller does mc_snum++ as above */
 		mc->mc_flags &= ~(C_INITIALIZED|C_EOF);
 		return rc;
@@ -6445,7 +6446,7 @@ skip:
 		mdb_xcursor_init1(mc, leaf);
 	}
 	if (data) {
-		if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS)
+		if ((rc = mdb_node_read(mc, leaf, data)) != MDB_SUCCESS)
 			return rc;
 
 		if (F_ISSET(leaf->mn_flags, F_DUPDATA)) {
@@ -6530,7 +6531,7 @@ mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
 		mdb_xcursor_init1(mc, leaf);
 	}
 	if (data) {
-		if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS)
+		if ((rc = mdb_node_read(mc, leaf, data)) != MDB_SUCCESS)
 			return rc;
 
 		if (F_ISSET(leaf->mn_flags, F_DUPDATA)) {
@@ -6717,7 +6718,7 @@ set1:
 		} else if (op == MDB_GET_BOTH || op == MDB_GET_BOTH_RANGE) {
 			MDB_val olddata;
 			MDB_cmp_func *dcmp;
-			if ((rc = mdb_node_read(mc->mc_txn, leaf, &olddata)) != MDB_SUCCESS)
+			if ((rc = mdb_node_read(mc, leaf, &olddata)) != MDB_SUCCESS)
 				return rc;
 			dcmp = mc->mc_dbx->md_dcmp;
 #if UINT_MAX < SIZE_MAX || defined(MDB_VL32)
@@ -6735,7 +6736,7 @@ set1:
 		} else {
 			if (mc->mc_xcursor)
 				mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF);
-			if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS)
+			if ((rc = mdb_node_read(mc, leaf, data)) != MDB_SUCCESS)
 				return rc;
 		}
 	}
@@ -6790,7 +6791,7 @@ mdb_cursor_first(MDB_cursor *mc, MDB_val *key, MDB_val *data)
 			if (rc)
 				return rc;
 		} else {
-			if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS)
+			if ((rc = mdb_node_read(mc, leaf, data)) != MDB_SUCCESS)
 				return rc;
 		}
 	}
@@ -6841,7 +6842,7 @@ mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data)
 			if (rc)
 				return rc;
 		} else {
-			if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS)
+			if ((rc = mdb_node_read(mc, leaf, data)) != MDB_SUCCESS)
 				return rc;
 		}
 	}
@@ -6887,7 +6888,7 @@ mdb_cursor_get(MDB_cursor *mc, MDB_val *key, MDB_val *data,
 					if (F_ISSET(leaf->mn_flags, F_DUPDATA)) {
 						rc = mdb_cursor_get(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_GET_CURRENT);
 					} else {
-						rc = mdb_node_read(mc->mc_txn, leaf, data);
+						rc = mdb_node_read(mc, leaf, data);
 					}
 				}
 			}
@@ -6993,7 +6994,7 @@ fetchm:
 			MDB_node *leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
 			if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) {
 				MDB_GET_KEY(leaf, key);
-				rc = mdb_node_read(mc->mc_txn, leaf, data);
+				rc = mdb_node_read(mc, leaf, data);
 				break;
 			}
 		}
@@ -7377,7 +7378,7 @@ current:
 			int level, ovpages, dpages = OVPAGES(data->mv_size, env->me_psize);
 
 			memcpy(&pg, olddata.mv_data, sizeof(pg));
-			if ((rc2 = mdb_page_get(mc->mc_txn, pg, &omp, &level)) != 0)
+			if ((rc2 = mdb_page_get(mc, pg, &omp, &level)) != 0)
 				return rc2;
 			ovpages = omp->mp_pages;
 
@@ -7678,7 +7679,7 @@ mdb_cursor_del(MDB_cursor *mc, unsigned int flags)
 		pgno_t pg;
 
 		memcpy(&pg, NODEDATA(leaf), sizeof(pg));
-		if ((rc = mdb_page_get(mc->mc_txn, pg, &omp, NULL)) ||
+		if ((rc = mdb_page_get(mc, pg, &omp, NULL)) ||
 			(rc = mdb_ovpage_free(mc, omp)))
 			goto fail;
 	}
@@ -8837,7 +8838,7 @@ mdb_rebalance(MDB_cursor *mc)
 			if (rc)
 				return rc;
 			mc->mc_db->md_root = NODEPGNO(NODEPTR(mp, 0));
-			rc = mdb_page_get(mc->mc_txn,mc->mc_db->md_root,&mc->mc_pg[0],NULL);
+			rc = mdb_page_get(mc, mc->mc_db->md_root, &mc->mc_pg[0], NULL);
 			if (rc)
 				return rc;
 			mc->mc_db->md_depth--;
@@ -8898,7 +8899,7 @@ mdb_rebalance(MDB_cursor *mc)
 		DPUTS("reading right neighbor");
 		mn.mc_ki[ptop]++;
 		node = NODEPTR(mc->mc_pg[ptop], mn.mc_ki[ptop]);
-		rc = mdb_page_get(mc->mc_txn,NODEPGNO(node),&mn.mc_pg[mn.mc_top],NULL);
+		rc = mdb_page_get(mc, NODEPGNO(node), &mn.mc_pg[mn.mc_top], NULL);
 		if (rc)
 			return rc;
 		mn.mc_ki[mn.mc_top] = 0;
@@ -8910,7 +8911,7 @@ mdb_rebalance(MDB_cursor *mc)
 		DPUTS("reading left neighbor");
 		mn.mc_ki[ptop]--;
 		node = NODEPTR(mc->mc_pg[ptop], mn.mc_ki[ptop]);
-		rc = mdb_page_get(mc->mc_txn,NODEPGNO(node),&mn.mc_pg[mn.mc_top],NULL);
+		rc = mdb_page_get(mc, NODEPGNO(node), &mn.mc_pg[mn.mc_top], NULL);
 		if (rc)
 			return rc;
 		mn.mc_ki[mn.mc_top] = NUMKEYS(mn.mc_pg[mn.mc_top]) - 1;
@@ -9652,7 +9653,6 @@ static int ESECT
 mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags)
 {
 	MDB_cursor mc;
-	MDB_txn *txn = my->mc_txn;
 	MDB_node *ni;
 	MDB_page *mo, *mp, *leaf;
 	char *buf, *ptr;
@@ -9665,9 +9665,9 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags)
 
 	mc.mc_snum = 1;
 	mc.mc_top = 0;
-	mc.mc_txn = txn;
+	mc.mc_txn = my->mc_txn;
 
-	rc = mdb_page_get(my->mc_txn, *pg, &mc.mc_pg[0], NULL);
+	rc = mdb_page_get(&mc, *pg, &mc.mc_pg[0], NULL);
 	if (rc)
 		return rc;
 	rc = mdb_page_search_root(&mc, NULL, MDB_PS_FIRST);
@@ -9711,7 +9711,7 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags)
 						}
 
 						memcpy(&pg, NODEDATA(ni), sizeof(pg));
-						rc = mdb_page_get(txn, pg, &omp, NULL);
+						rc = mdb_page_get(&mc, pg, &omp, NULL);
 						if (rc)
 							goto done;
 						if (my->mc_wlen[toggle] >= MDB_WBUF) {
@@ -9762,7 +9762,7 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags)
 again:
 				ni = NODEPTR(mp, mc.mc_ki[mc.mc_top]);
 				pg = NODEPGNO(ni);
-				rc = mdb_page_get(txn, pg, &mp, NULL);
+				rc = mdb_page_get(&mc, pg, &mp, NULL);
 				if (rc)
 					goto done;
 				mc.mc_top++;
@@ -10435,7 +10435,7 @@ mdb_drop0(MDB_cursor *mc, int subs)
 #ifdef MDB_VL32
 		/* bump refcount for mx's pages */
 		for (i=0; i<mc->mc_snum; i++)
-			mdb_page_get(txn, mc->mc_pg[i]->mp_pgno, &mx.mc_pg[i], NULL);
+			mdb_page_get(&mx, mc->mc_pg[i]->mp_pgno, &mx.mc_pg[i], NULL);
 #endif
 		while (mc->mc_snum > 0) {
 			MDB_page *mp = mc->mc_pg[mc->mc_top];
@@ -10447,7 +10447,7 @@ mdb_drop0(MDB_cursor *mc, int subs)
 						MDB_page *omp;
 						pgno_t pg;
 						memcpy(&pg, NODEDATA(ni), sizeof(pg));
-						rc = mdb_page_get(txn, pg, &omp, NULL);
+						rc = mdb_page_get(mc, pg, &omp, NULL);
 						if (rc != 0)
 							goto done;
 						mdb_cassert(mc, IS_OVERFLOW(omp));

From 33025182ccfde2c9ac0073bbf3946398267ecc4d Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sun, 23 Aug 2015 21:10:52 +0200
Subject: [PATCH 196/504] Prep for next commit: C_WRITEMAP, C_ORIG_RDONLY.

No change in behavior.
---
 libraries/liblmdb/mdb.c | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 71e20a3c59..d2b1d57199 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1298,6 +1298,10 @@ struct MDB_cursor {
 #define C_SUB	0x04			/**< Cursor is a sub-cursor */
 #define C_DEL	0x08			/**< last op was a cursor_del */
 #define C_UNTRACK	0x40		/**< Un-track cursor when closing */
+#define C_WRITEMAP	MDB_TXN_WRITEMAP /**< Copy of txn flag */
+/** Read-only cursor into the txn's original snapshot in the map.
+ */
+#define C_ORIG_RDONLY	MDB_TXN_RDONLY
 /** @} */
 	unsigned int	mc_flags;	/**< @ref mdb_cursor */
 	MDB_page	*mc_pg[CURSOR_STACK];	/**< stack of pushed pages */
@@ -5915,7 +5919,7 @@ mdb_page_get(MDB_cursor *mc, pgno_t pgno, MDB_page **ret, int *lvl)
 	MDB_page *p = NULL;
 	int level;
 
-	if (! (txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_WRITEMAP))) {
+	if (! (mc->mc_flags & (C_ORIG_RDONLY|C_WRITEMAP))) {
 		MDB_txn *tx2 = txn;
 		level = 1;
 		do {
@@ -8038,7 +8042,7 @@ mdb_xcursor_init0(MDB_cursor *mc)
 	mx->mx_cursor.mc_dbflag = &mx->mx_dbflag;
 	mx->mx_cursor.mc_snum = 0;
 	mx->mx_cursor.mc_top = 0;
-	mx->mx_cursor.mc_flags = C_SUB;
+	mx->mx_cursor.mc_flags = C_SUB | (mc->mc_flags & (C_ORIG_RDONLY|C_WRITEMAP));
 	mx->mx_dbx.md_name.mv_size = 0;
 	mx->mx_dbx.md_name.mv_data = NULL;
 	mx->mx_dbx.md_cmp = mc->mc_dbx->md_dcmp;
@@ -8057,12 +8061,12 @@ mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node)
 {
 	MDB_xcursor *mx = mc->mc_xcursor;
 
+	mx->mx_cursor.mc_flags &= C_SUB|C_ORIG_RDONLY|C_WRITEMAP;
 	if (node->mn_flags & F_SUBDATA) {
 		memcpy(&mx->mx_db, NODEDATA(node), sizeof(MDB_db));
 		mx->mx_cursor.mc_pg[0] = 0;
 		mx->mx_cursor.mc_snum = 0;
 		mx->mx_cursor.mc_top = 0;
-		mx->mx_cursor.mc_flags = C_SUB;
 	} else {
 		MDB_page *fp = NODEDATA(node);
 		mx->mx_db.md_pad = 0;
@@ -8075,7 +8079,7 @@ mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node)
 		COPY_PGNO(mx->mx_db.md_root, fp->mp_pgno);
 		mx->mx_cursor.mc_snum = 1;
 		mx->mx_cursor.mc_top = 0;
-		mx->mx_cursor.mc_flags = C_INITIALIZED|C_SUB;
+		mx->mx_cursor.mc_flags |= C_INITIALIZED;
 		mx->mx_cursor.mc_pg[0] = fp;
 		mx->mx_cursor.mc_ki[0] = 0;
 		if (mc->mc_db->md_flags & MDB_DUPFIXED) {
@@ -8141,7 +8145,7 @@ mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx)
 	mc->mc_top = 0;
 	mc->mc_pg[0] = 0;
 	mc->mc_ki[0] = 0;
-	mc->mc_flags = 0;
+	mc->mc_flags = txn->mt_flags & (C_ORIG_RDONLY|C_WRITEMAP);
 	if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) {
 		mdb_tassert(txn, mx != NULL);
 		mc->mc_xcursor = mx;

From 6534b804a57d5f6cfc40ca7f63a64d1c5153748f Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sun, 23 Aug 2015 22:00:14 +0200
Subject: [PATCH 197/504] mdb_page_alloc(): Use original snapshot of freeDB.

I can't help thinking this should be safer, and useful somewhere.
---
 libraries/liblmdb/mdb.c | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index d2b1d57199..67c2f2d5bb 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1300,6 +1300,8 @@ struct MDB_cursor {
 #define C_UNTRACK	0x40		/**< Un-track cursor when closing */
 #define C_WRITEMAP	MDB_TXN_WRITEMAP /**< Copy of txn flag */
 /** Read-only cursor into the txn's original snapshot in the map.
+ *	Set for read-only txns, and in #mdb_page_alloc() for #FREE_DBI when
+ *	#MDB_DEVEL & 2. Only implements code which is necessary for this.
  */
 #define C_ORIG_RDONLY	MDB_TXN_RDONLY
 /** @} */
@@ -2259,6 +2261,8 @@ mdb_page_dirty(MDB_txn *txn, MDB_page *mp)
  * Do not modify the freedB, just merge freeDB records into me_pghead[]
  * and move me_pglast to say which records were consumed.  Only this
  * function can create me_pghead and move me_pglast/mt_next_pgno.
+ * When #MDB_DEVEL & 2, it is not affected by #mdb_freelist_save(): it
+ * then uses the transaction's original snapshot of the freeDB.
  * @param[in] mc cursor A cursor handle identifying the transaction and
  *	database for which we are allocating.
  * @param[in] num the number of pages to allocate.
@@ -2334,6 +2338,14 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
 			last = env->me_pglast;
 			oldest = env->me_pgoldest;
 			mdb_cursor_init(&m2, txn, FREE_DBI, NULL);
+#if (MDB_DEVEL) & 2	/* "& 2" so MDB_DEVEL=1 won't hide bugs breaking freeDB */
+			/* Use original snapshot. TODO: Should need less care in code
+			 * which modifies the database. Maybe we can delete some code?
+			 */
+			m2.mc_flags |= C_ORIG_RDONLY;
+			m2.mc_db = &env->me_metas[(txn->mt_txnid-1) & 1]->mm_dbs[FREE_DBI];
+			m2.mc_dbflag = (unsigned char *)""; /* probably unnecessary */
+#endif
 			if (last) {
 				op = MDB_SET_RANGE;
 				key.mv_data = &last; /* will look up last+1 */
@@ -3260,6 +3272,9 @@ mdb_txn_abort(MDB_txn *txn)
 
 /** Save the freelist as of this transaction to the freeDB.
  * This changes the freelist. Keep trying until it stabilizes.
+ *
+ * When (MDB_DEVEL) & 2, the changes do not affect #mdb_page_alloc(),
+ * it then uses the transaction's original snapshot of the freeDB.
  */
 static int
 mdb_freelist_save(MDB_txn *txn)

From 3a2ac24f5bbdee304e1772ec71f81a4b6f8be87e Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 22 Dec 2015 04:22:01 +0000
Subject: [PATCH 198/504] MDB_VL32 - track overflow pages too

Otherwise they'll fill up the txn's pageref list when traversing
a DB with lots of overflow records.
---
 libraries/liblmdb/mdb.c | 30 ++++++++++++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 67c2f2d5bb..698797feab 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1308,6 +1308,9 @@ struct MDB_cursor {
 	unsigned int	mc_flags;	/**< @ref mdb_cursor */
 	MDB_page	*mc_pg[CURSOR_STACK];	/**< stack of pushed pages */
 	indx_t		mc_ki[CURSOR_STACK];	/**< stack of page indices */
+#ifdef MDB_VL32
+	MDB_page	*mc_ovpg;		/**< a referenced overflow page */
+#endif
 };
 
 	/** Context for sorted-dup records.
@@ -1953,6 +1956,10 @@ mdb_cursor_unref(MDB_cursor *mc)
 		return;
 	for (i=0; i<mc->mc_snum; i++)
 		mdb_page_unref(mc->mc_txn, mc->mc_pg[i]);
+	if (mc->mc_ovpg) {
+		mdb_page_unref(mc->mc_txn, mc->mc_ovpg);
+		mc->mc_ovpg = 0;
+	}
 	mc->mc_snum = mc->mc_top = 0;
 	mc->mc_pg[0] = NULL;
 	mc->mc_flags &= ~C_INITIALIZED;
@@ -5655,8 +5662,9 @@ mdb_cursor_push(MDB_cursor *mc, MDB_page *mp)
  * The caller must copy the data if it must be used later in the same txn.
  *
  * Also - our reference counting revolves around cursors, but overflow pages
- * aren't ever pointed to by a cursor. As such, they always remain referenced
- * in a txn until it ends.
+ * aren't pointed to by a cursor's page stack. We have to remember them
+ * explicitly, in the added mc_ovpg field. A single cursor can only hold a
+ * reference to one overflow page at a time.
  *
  * @param[in] txn the transaction for this access.
  * @param[in] pgno the page number for the page to retrieve.
@@ -6277,6 +6285,12 @@ mdb_node_read(MDB_cursor *mc, MDB_node *leaf, MDB_val *data)
 	pgno_t		 pgno;
 	int rc;
 
+#ifdef MDB_VL32
+	if (mc->mc_ovpg) {
+		MDB_PAGE_UNREF(mc->mc_txn, mc->mc_ovpg);
+		mc->mc_ovpg = 0;
+	}
+#endif
 	if (!F_ISSET(leaf->mn_flags, F_BIGDATA)) {
 		data->mv_size = NODEDSZ(leaf);
 		data->mv_data = NODEDATA(leaf);
@@ -6292,6 +6306,9 @@ mdb_node_read(MDB_cursor *mc, MDB_node *leaf, MDB_val *data)
 		return rc;
 	}
 	data->mv_data = METADATA(omp);
+#ifdef MDB_VL32
+	mc->mc_ovpg = omp;
+#endif
 
 	return MDB_SUCCESS;
 }
@@ -8057,6 +8074,9 @@ mdb_xcursor_init0(MDB_cursor *mc)
 	mx->mx_cursor.mc_dbflag = &mx->mx_dbflag;
 	mx->mx_cursor.mc_snum = 0;
 	mx->mx_cursor.mc_top = 0;
+#ifdef MDB_VL32
+	mx->mx_cursor.mc_ovpg = 0;
+#endif
 	mx->mx_cursor.mc_flags = C_SUB | (mc->mc_flags & (C_ORIG_RDONLY|C_WRITEMAP));
 	mx->mx_dbx.md_name.mv_size = 0;
 	mx->mx_dbx.md_name.mv_data = NULL;
@@ -8160,6 +8180,9 @@ mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx)
 	mc->mc_top = 0;
 	mc->mc_pg[0] = 0;
 	mc->mc_ki[0] = 0;
+#ifdef MDB_VL32
+	mc->mc_ovpg = 0;
+#endif
 	mc->mc_flags = txn->mt_flags & (C_ORIG_RDONLY|C_WRITEMAP);
 	if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) {
 		mdb_tassert(txn, mx != NULL);
@@ -8774,6 +8797,9 @@ mdb_cursor_copy(const MDB_cursor *csrc, MDB_cursor *cdst)
 	cdst->mc_snum = csrc->mc_snum;
 	cdst->mc_top = csrc->mc_top;
 	cdst->mc_flags = csrc->mc_flags;
+#ifdef MDB_VL32
+	cdst->mc_ovpg = csrc->mc_ovpg;
+#endif
 
 	for (i=0; i<csrc->mc_snum; i++) {
 		cdst->mc_pg[i] = csrc->mc_pg[i];

From 5eb25c5cb9122fcc016e770bcc1f033e3a9d2a0f Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@symas.com>
Date: Tue, 22 Dec 2015 05:20:33 +0000
Subject: [PATCH 199/504] MDB_VL32 - fix for Win32 read-only envs

We can't map with MEM_RESERVE because that requires write access
to the underlying file/section. Mapping with the default (MEM_COMMIT)
requires that we don't map past the end of the file.
---
 libraries/liblmdb/mdb.c | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 698797feab..516e69a7ec 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5689,7 +5689,7 @@ mdb_rpage_get(MDB_txn *txn, pgno_t pg0, MDB_page **ret)
 #define MAP(rc,env,addr,len,off)	\
 	addr = NULL; \
 	rc = NtMapViewOfSection(env->me_fmh, GetCurrentProcess(), &addr, 0, \
-		len, &off, &len, ViewUnmap, MEM_RESERVE, PAGE_READONLY)
+		len, &off, &len, ViewUnmap, (env->me_flags & MDB_RDONLY) ? 0 : MEM_RESERVE, PAGE_READONLY)
 #else
 	off_t off;
 	size_t len;
@@ -5803,9 +5803,13 @@ retry:
 		id3.mref = 1;
 		if (id3.mid)
 			goto found;
-		len = env->me_psize * MDB_RPAGE_CHUNK;
+		/* don't map past last written page */
+		if (pgno + MDB_RPAGE_CHUNK-1 > txn->mt_last_pgno)
+			id3.mcnt = txn->mt_last_pgno + 1 - pgno;
+		else
+			id3.mcnt = MDB_RPAGE_CHUNK;
+		len = id3.mcnt * env->me_psize;
 		id3.mid = pgno;
-		id3.mcnt = MDB_RPAGE_CHUNK;
 
 		/* search for page in env */
 		LOCK_MUTEX0(env->me_rpmutex);

From 9d75a82ae10fdcee80b8b8e82c6ef9b6ab83dc47 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 22 Dec 2015 06:13:25 +0000
Subject: [PATCH 200/504] MDB_VL32 - fix prev commit

Only tweak length for read-only envs
---
 libraries/liblmdb/mdb.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 516e69a7ec..4ff49500a5 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5803,8 +5803,8 @@ retry:
 		id3.mref = 1;
 		if (id3.mid)
 			goto found;
-		/* don't map past last written page */
-		if (pgno + MDB_RPAGE_CHUNK-1 > txn->mt_last_pgno)
+		/* don't map past last written page in read-only envs */
+		if ((env->me_flags & MDB_RDONLY) && pgno + MDB_RPAGE_CHUNK-1 > txn->mt_last_pgno)
 			id3.mcnt = txn->mt_last_pgno + 1 - pgno;
 		else
 			id3.mcnt = MDB_RPAGE_CHUNK;

From 825ab2ad52ffe5038ccda209aa05d9917915f6ff Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 22 Dec 2015 18:30:41 +0000
Subject: [PATCH 201/504] MDB_RESERVE doc

Add mdb_put text to mdb_cursor_put description for people who
fail to draw logical conclusions.
---
 libraries/liblmdb/lmdb.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index ff19c0afe8..afcfe889c7 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -1470,7 +1470,8 @@ int  mdb_cursor_get(MDB_cursor *cursor, MDB_val *key, MDB_val *data,
 	 *		the database supports duplicates (#MDB_DUPSORT).
 	 *	<li>#MDB_RESERVE - reserve space for data of the given size, but
 	 *		don't copy the given data. Instead, return a pointer to the
-	 *		reserved space, which the caller can fill in later. This saves
+	 *		reserved space, which the caller can fill in later - before
+	 *		the next update operation or the transaction ends. This saves
 	 *		an extra memcpy if the data is being generated later. This flag
 	 *		must not be specified if the database was opened with #MDB_DUPSORT.
 	 *	<li>#MDB_APPEND - append the given key/data pair to the end of the

From 90d1ee279fca1b26daf0223377a44a6b17f46ad0 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 22 Dec 2015 20:24:26 +0000
Subject: [PATCH 202/504] MDB_VL32 - Fix me_rpmutex usage

---
 libraries/liblmdb/mdb.c | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 4ff49500a5..56243dc7f2 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1403,7 +1403,7 @@ struct MDB_env {
 #endif
 #ifdef MDB_VL32
 	MDB_ID3L	me_rpages;	/**< like #mt_rpages, but global to env */
-	mdb_mutex_t	me_rpmutex;	/**< control access to #me_rpages */
+	pthread_mutex_t	me_rpmutex;	/**< control access to #me_rpages */
 #define MDB_ERPAGE_SIZE	16384
 #define MDB_ERPAGE_MAX	(MDB_ERPAGE_SIZE-1)
 	unsigned int me_rpcheck;
@@ -3227,7 +3227,7 @@ mdb_txn_end(MDB_txn *txn, unsigned mode)
 	if (!txn->mt_parent) {
 		MDB_ID3L el = env->me_rpages, tl = txn->mt_rpages;
 		unsigned i, x, n = tl[0].mid;
-		LOCK_MUTEX0(env->me_rpmutex);
+		pthread_mutex_lock(&env->me_rpmutex);
 		for (i = 1; i <= n; i++) {
 			if (tl[i].mid & (MDB_RPAGE_CHUNK-1)) {
 				/* tmp overflow pages that we didn't share in env */
@@ -3242,7 +3242,7 @@ mdb_txn_end(MDB_txn *txn, unsigned mode)
 				}
 			}
 		}
-		UNLOCK_MUTEX(env->me_rpmutex);
+		pthread_mutex_unlock(&env->me_rpmutex);
 		tl[0].mid = 0;
 		if (mode & MDB_END_FREE)
 			free(tl);
@@ -5016,7 +5016,7 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 #ifdef _WIN32
 	env->me_rpmutex = CreateMutex(NULL, FALSE, NULL);
 #else
-	pthread_mutex_init(env->me_rpmutex, NULL);
+	pthread_mutex_init(&env->me_rpmutex, NULL);
 #endif
 #endif
 
@@ -5352,7 +5352,7 @@ mdb_env_close0(MDB_env *env, int excl)
 #ifdef _WIN32
 	if (env->me_rpmutex) CloseHandle(env->me_rpmutex);
 #else
-	pthread_mutex_destroy(env->me_rpmutex);
+	pthread_mutex_destroy(&env->me_rpmutex);
 #endif
 #endif
 
@@ -5735,7 +5735,7 @@ mdb_rpage_get(MDB_txn *txn, pgno_t pg0, MDB_page **ret)
 				/* if no active ref, see if we can replace in env */
 				if (!tl[x].mref) {
 					unsigned i;
-					LOCK_MUTEX0(env->me_rpmutex);
+					pthread_mutex_lock(&env->me_rpmutex);
 					i = mdb_mid3l_search(el, tl[x].mid);
 					if (el[i].mref == 1) {
 						/* just us, replace it */
@@ -5746,7 +5746,7 @@ mdb_rpage_get(MDB_txn *txn, pgno_t pg0, MDB_page **ret)
 						/* there are others, remove ourself */
 						el[i].mref--;
 					}
-					UNLOCK_MUTEX(env->me_rpmutex);
+					pthread_mutex_unlock(&env->me_rpmutex);
 				}
 			}
 		}
@@ -5760,7 +5760,7 @@ notlocal:
 	if (tl[0].mid >= MDB_TRPAGE_MAX - txn->mt_rpcheck) {
 		unsigned i, y;
 		/* purge unref'd pages from our list and unref in env */
-		LOCK_MUTEX0(env->me_rpmutex);
+		pthread_mutex_lock(&env->me_rpmutex);
 retry:
 		y = 0;
 		for (i=1; i<tl[0].mid; i++) {
@@ -5775,7 +5775,7 @@ retry:
 				el[x].mref--;
 			}
 		}
-		UNLOCK_MUTEX(env->me_rpmutex);
+		pthread_mutex_unlock(&env->me_rpmutex);
 		if (!y) {
 			/* we didn't find any unref'd chunks.
 			 * if we're out of room, fail.
@@ -5812,7 +5812,7 @@ retry:
 		id3.mid = pgno;
 
 		/* search for page in env */
-		LOCK_MUTEX0(env->me_rpmutex);
+		pthread_mutex_lock(&env->me_rpmutex);
 		x = mdb_mid3l_search(el, pgno);
 		if (x <= el[0].mid && el[x].mid == pgno) {
 			id3.mptr = el[x].mptr;
@@ -5832,12 +5832,12 @@ retry:
 					el[x].mcnt = id3.mcnt;
 				} else {
 					id3.mid = pg0;
-					UNLOCK_MUTEX(env->me_rpmutex);
+					pthread_mutex_unlock(&env->me_rpmutex);
 					goto found;
 				}
 			}
 			el[x].mref++;
-			UNLOCK_MUTEX(env->me_rpmutex);
+			pthread_mutex_unlock(&env->me_rpmutex);
 			goto found;
 		}
 		if (el[0].mid >= MDB_ERPAGE_MAX - env->me_rpcheck) {
@@ -5857,7 +5857,7 @@ retry:
 					goto retry;
 				}
 				if (el[0].mid >= MDB_ERPAGE_MAX) {
-					UNLOCK_MUTEX(env->me_rpmutex);
+					pthread_mutex_unlock(&env->me_rpmutex);
 					return MDB_MAP_FULL;
 				}
 				env->me_rpcheck /= 2;
@@ -5876,7 +5876,7 @@ retry:
 		MAP(rc, env, id3.mptr, len, off);
 		if (rc) {
 fail:
-			UNLOCK_MUTEX(env->me_rpmutex);
+			pthread_mutex_unlock(&env->me_rpmutex);
 			return rc;
 		}
 		/* If this page is far enough from the end of the env, scan for
@@ -5911,7 +5911,7 @@ fail:
 			}
 		}
 		mdb_mid3l_insert(el, &id3);
-		UNLOCK_MUTEX(env->me_rpmutex);
+		pthread_mutex_unlock(&env->me_rpmutex);
 found:
 		mdb_mid3l_insert(tl, &id3);
 	} else {

From 7b9928ced4022b09cea3369c93f5bcdf4008076c Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 23 Dec 2015 23:02:06 +0000
Subject: [PATCH 203/504] MDB_VL32 - prevent mixing with non-VL32 builds

---
 libraries/liblmdb/lmdb.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index afcfe889c7..587a44eaf4 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -176,6 +176,7 @@ typedef	mode_t	mdb_mode_t;
 
 #ifdef MDB_VL32
 typedef uint64_t	mdb_size_t;
+#define mdb_env_create(env)	mdb_env_create_vl32(env)	/**< Prevent mixing with non-VL32 builds */
 #else
 typedef size_t	mdb_size_t;
 #endif

From c3852f29af4c38ba7c91d7d3956577f980885cab Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 28 Dec 2015 19:37:42 +0000
Subject: [PATCH 204/504] MDB_VL32 tweak prev commit

---
 libraries/liblmdb/lmdb.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 587a44eaf4..d616cabf95 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -176,7 +176,7 @@ typedef	mode_t	mdb_mode_t;
 
 #ifdef MDB_VL32
 typedef uint64_t	mdb_size_t;
-#define mdb_env_create(env)	mdb_env_create_vl32(env)	/**< Prevent mixing with non-VL32 builds */
+#define mdb_env_create	mdb_env_create_vl32	/**< Prevent mixing with non-VL32 builds */
 #else
 typedef size_t	mdb_size_t;
 #endif

From 1c2a5888cfffdfb3976145634ee5663e4b621675 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 28 Dec 2015 19:33:51 +0000
Subject: [PATCH 205/504] ITS#8342 MDB_VL32/WIN32 - close file mapping handle
 in env_close

---
 libraries/liblmdb/mdb.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 56243dc7f2..19930fcfc7 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5350,6 +5350,7 @@ mdb_env_close0(MDB_env *env, int excl)
 	}
 #ifdef MDB_VL32
 #ifdef _WIN32
+	if (env->me_fmh) CloseHandle(env->me_fmh);
 	if (env->me_rpmutex) CloseHandle(env->me_rpmutex);
 #else
 	pthread_mutex_destroy(&env->me_rpmutex);

From 53f624bff0d7d08b64b48ab0c6683f4145b9bcac Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 28 Dec 2015 20:31:09 +0000
Subject: [PATCH 206/504] MDB_VL32 fix cursor_unref - ignore cursor with empty
 stack

---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 19930fcfc7..ae32c98961 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1952,7 +1952,7 @@ static void
 mdb_cursor_unref(MDB_cursor *mc)
 {
 	int i;
-	if (!mc->mc_pg[0] || IS_SUBP(mc->mc_pg[0]))
+	if (!mc->mc_snum || !mc->mc_pg[0] || IS_SUBP(mc->mc_pg[0]))
 		return;
 	for (i=0; i<mc->mc_snum; i++)
 		mdb_page_unref(mc->mc_txn, mc->mc_pg[i]);

From 5db0b54ca127eda9777771b3a5e3c956491e7660 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@symas.com>
Date: Sat, 2 Jan 2016 12:19:42 +0000
Subject: [PATCH 207/504] Fixups for env_copy with large files

wsize was being truncated to 32bits on Windows.
Only try to write 1GB at a time on Windows64;
larger writes fail with ERROR_WORKING_SET_QUOTA.
---
 libraries/liblmdb/mdb.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index ae32c98961..09f5132599 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1426,7 +1426,7 @@ typedef struct MDB_ntxn {
 #endif
 
 	/** max bytes to write in one call */
-#define MAX_WRITE		(0x80000000U >> (sizeof(ssize_t) == 4))
+#define MAX_WRITE		(0x40000000U >> (sizeof(ssize_t) == 4))
 
 	/** Check \b txn and \b dbi arguments to a function */
 #define TXN_DBI_EXIST(txn, dbi, validity) \
@@ -9977,7 +9977,7 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd)
 	MDB_txn *txn = NULL;
 	mdb_mutexref_t wmutex = NULL;
 	int rc;
-	size_t wsize;
+	mdb_size_t wsize, w3;
 	char *ptr;
 #ifdef _WIN32
 	DWORD len, w2;
@@ -10036,15 +10036,15 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd)
 	if (rc)
 		goto leave;
 
-	w2 = txn->mt_next_pgno * env->me_psize;
+	w3 = txn->mt_next_pgno * env->me_psize;
 	{
 		mdb_size_t fsize = 0;
 		if ((rc = mdb_fsize(env->me_fd, &fsize)))
 			goto leave;
-		if (w2 > fsize)
-			w2 = fsize;
+		if (w3 > fsize)
+			w3 = fsize;
 	}
-	wsize = w2 - wsize;
+	wsize = w3 - wsize;
 	while (wsize > 0) {
 		if (wsize > MAX_WRITE)
 			w2 = MAX_WRITE;

From 27b1c5f366dca98cfe6924dec5d7c989fbd91dba Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 7 Jan 2016 18:28:29 +0000
Subject: [PATCH 208/504] Add MDB_PREV_MULTIPLE

Logical counterpart to GET_MULTIPLE, NEXT_MULTIPLE
---
 libraries/liblmdb/lmdb.h |  4 +++-
 libraries/liblmdb/mdb.c  | 22 ++++++++++++++++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index d616cabf95..98e8c5127d 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -392,7 +392,9 @@ typedef enum MDB_cursor_op {
 	MDB_PREV_NODUP,			/**< Position at last data item of previous key */
 	MDB_SET,				/**< Position at specified key */
 	MDB_SET_KEY,			/**< Position at specified key, return key + data */
-	MDB_SET_RANGE			/**< Position at first key greater than or equal to specified key. */
+	MDB_SET_RANGE,			/**< Position at first key greater than or equal to specified key. */
+	MDB_PREV_MULTIPLE		/**< Position at previous page and return key and up to
+								a page of duplicate data items. Only for #MDB_DUPFIXED */
 } MDB_cursor_op;
 
 /** @defgroup  errors	Return Codes
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 09f5132599..5ef095f719 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -6997,6 +6997,28 @@ fetchm:
 			}
 		}
 		break;
+	case MDB_PREV_MULTIPLE:
+		if (data == NULL) {
+			rc = EINVAL;
+			break;
+		}
+		if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) {
+			rc = MDB_INCOMPATIBLE;
+			break;
+		}
+		if (!(mc->mc_flags & C_INITIALIZED))
+			rc = mdb_cursor_first(mc, key, data);
+		else {
+			MDB_cursor *mx = &mc->mc_xcursor->mx_cursor;
+			if (mx->mc_flags & C_INITIALIZED) {
+				rc = mdb_cursor_sibling(mx, 0);
+				if (rc == MDB_SUCCESS)
+					goto fetchm;
+			} else {
+				rc = MDB_NOTFOUND;
+			}
+		}
+		break;
 	case MDB_NEXT:
 	case MDB_NEXT_DUP:
 	case MDB_NEXT_NODUP:

From 447683a8244dccdd395d33fa9cf45ac23d10cfc3 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 11 Jan 2016 20:17:42 +0000
Subject: [PATCH 209/504] ITS#8346 free last txn0->mt_rpages in env_close

---
 libraries/liblmdb/mdb.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 5ef095f719..bd129c4c2d 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5255,14 +5255,16 @@ mdb_env_close0(MDB_env *env, int excl)
 	free(env->me_dbflags);
 	free(env->me_path);
 	free(env->me_dirty_list);
-	free(env->me_txn0);
 #ifdef MDB_VL32
+	if (env->me_txn0 && env->me_txn->mt_rpages)
+		free(env->me_txn0->mt_rpages);
 	{ unsigned int x;
 		for (x=1; x<=env->me_rpages[0].mid; x++)
 		munmap(env->me_rpages[x].mptr, env->me_rpages[x].mcnt * env->me_psize);
 	}
 	free(env->me_rpages);
 #endif
+	free(env->me_txn0);
 	mdb_midl_free(env->me_free_pgs);
 
 	if (env->me_flags & MDB_ENV_TXKEY) {

From a5bf1648c69d316a5df0d45b2f43fc61f071d043 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 11 Jan 2016 20:25:33 +0000
Subject: [PATCH 210/504] ITS#8347 fix off-by-one in VL32 purge

---
 libraries/liblmdb/mdb.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index bd129c4c2d..1991f01da3 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5766,7 +5766,7 @@ notlocal:
 		pthread_mutex_lock(&env->me_rpmutex);
 retry:
 		y = 0;
-		for (i=1; i<tl[0].mid; i++) {
+		for (i=1; i<=tl[0].mid; i++) {
 			if (!tl[i].mref) {
 				if (!y) y = i;
 				/* tmp overflow pages don't go to env */
@@ -5846,7 +5846,7 @@ retry:
 		if (el[0].mid >= MDB_ERPAGE_MAX - env->me_rpcheck) {
 			/* purge unref'd pages */
 			unsigned i, y = 0;
-			for (i=1; i<el[0].mid; i++) {
+			for (i=1; i<=el[0].mid; i++) {
 				if (!el[i].mref) {
 					if (!y) y = i;
 					munmap(el[i].mptr, env->me_psize * el[i].mcnt);

From 15880014baaaa892d33d42f4cc466ed8a18e1614 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 12 Jan 2016 07:13:48 +0000
Subject: [PATCH 211/504] ITS#8346 fix typo in prev commit

---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 1991f01da3..16544d5672 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5256,7 +5256,7 @@ mdb_env_close0(MDB_env *env, int excl)
 	free(env->me_path);
 	free(env->me_dirty_list);
 #ifdef MDB_VL32
-	if (env->me_txn0 && env->me_txn->mt_rpages)
+	if (env->me_txn0 && env->me_txn0->mt_rpages)
 		free(env->me_txn0->mt_rpages);
 	{ unsigned int x;
 		for (x=1; x<=env->me_rpages[0].mid; x++)

From d6995599b4a984171ec371633ac729f79d73b405 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 12 Jan 2016 23:18:06 +0100
Subject: [PATCH 212/504] lmdb.h Caveats: Reserved vs. actual mem/disk usage

---
 libraries/liblmdb/lmdb.h | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 98e8c5127d..f684da7368 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -78,6 +78,11 @@
  *	  access to locks and lock file. Exceptions: On read-only filesystems
  *	  or with the #MDB_NOLOCK flag described under #mdb_env_open().
  *
+ *	- An LMDB configuration will often reserve considerable \b unused
+ *	  memory address space and maybe file size for future growth.
+ *	  This does not use actual memory or disk space, but users may need
+ *	  to understand the difference so they won't be scared off.
+ *
  *	- By default, in versions before 0.9.10, unused portions of the data
  *	  file might receive garbage data from memory freed by other code.
  *	  (This does not happen when using the #MDB_WRITEMAP flag.) As of

From a6ccef73ed288271f9b5871909d14a2e481c81ae Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sat, 16 Jan 2016 17:11:36 +0000
Subject: [PATCH 213/504] ITS#8324 fix c4e31434c7773ee54f2ffdeb545e5740f56492a1

Actually, there is no guarantee that Windows will map newly written
data, so we need VirtualAlloc even for non-WRITEMAP.
---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 16544d5672..e3eef1cc21 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2429,7 +2429,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
 			goto fail;
 	}
 #ifdef _WIN32
-	if (env->me_flags & MDB_WRITEMAP) {
+	if (!(env->me_flags & MDB_RDONLY)) {
 		void *p;
 		p = (MDB_page *)(env->me_map + env->me_psize * pgno);
 		p = VirtualAlloc(p, env->me_psize * num, MEM_COMMIT,

From 20dec1f69bf4860202c764ce92b1fbbe3d11a065 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 20 Jan 2016 01:30:57 +0000
Subject: [PATCH 214/504] WIN64 needs off_t redefined too

---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index e3eef1cc21..895ab952b6 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -35,7 +35,7 @@
 #ifndef _GNU_SOURCE
 #define _GNU_SOURCE 1
 #endif
-#ifdef MDB_VL32
+#if defined(MDB_VL32) || defined(__WIN64__)
 #define _FILE_OFFSET_BITS	64
 #endif
 #ifdef _WIN32

From b5018e26bc155baea94a3552696ea9b8f0dc7046 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sat, 23 Jan 2016 16:51:42 +0000
Subject: [PATCH 215/504] Update WRITEMAP doc

---
 libraries/liblmdb/lmdb.h | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index f684da7368..7bcdb9c704 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -545,9 +545,11 @@ int  mdb_env_create(MDB_env **env);
 	 *		allowed. LMDB will still modify the lock file - except on read-only
 	 *		filesystems, where LMDB does not use locks.
 	 *	<li>#MDB_WRITEMAP
-	 *		Use a writeable memory map unless MDB_RDONLY is set. This is faster
-	 *		and uses fewer mallocs, but loses protection from application bugs
+	 *		Use a writeable memory map unless MDB_RDONLY is set. This uses
+	 *		fewer mallocs but loses protection from application bugs
 	 *		like wild pointer writes and other bad updates into the database.
+	 *		This may be slightly faster for DBs that fit entirely in RAM, but
+	 *		is slower for DBs larger than RAM.
 	 *		Incompatible with nested transactions.
 	 *		Do not mix processes with and without MDB_WRITEMAP on the same
 	 *		environment.  This can defeat durability (#mdb_env_sync etc).

From 8c215aa970215a58ee0df458813c0405ad27a6e9 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sun, 24 Jan 2016 11:32:49 +0000
Subject: [PATCH 216/504] ITS#8355 fix subcursors

make sure C_DEL gets reset in subcursor after it moves.
---
 libraries/liblmdb/mdb.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 895ab952b6..fa5416f436 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -6458,8 +6458,10 @@ mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
 
 	DPRINTF(("cursor_next: top page is %"Y"u in cursor %p",
 		mdb_dbg_pgno(mp), (void *) mc));
-	if (mc->mc_flags & C_DEL)
+	if (mc->mc_flags & C_DEL) {
+		mc->mc_flags ^= C_DEL;
 		goto skip;
+	}
 
 	if (mc->mc_ki[mc->mc_top] + 1u >= NUMKEYS(mp)) {
 		DPUTS("=====> move to next sibling page");
@@ -6545,6 +6547,8 @@ mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
 	DPRINTF(("cursor_prev: top page is %"Y"u in cursor %p",
 		mdb_dbg_pgno(mp), (void *) mc));
 
+	mc->mc_flags &= ~(C_EOF|C_DEL);
+
 	if (mc->mc_ki[mc->mc_top] == 0)  {
 		DPUTS("=====> move to prev sibling page");
 		if ((rc = mdb_cursor_sibling(mc, 0)) != MDB_SUCCESS) {

From 8f88b1b0ba208c51da70736626aeeb5daac6e3f2 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 26 Jan 2016 22:13:01 +0000
Subject: [PATCH 217/504] ITS#8324 fix a6ccef73ed288271f9b5871909d14a2e481c81ae

Removing the WRITEMAP test dropped this into the MDB_VL32 code path,
which was wrong.
---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index fa5416f436..be36d4967e 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2428,7 +2428,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
 			rc = MDB_MAP_FULL;
 			goto fail;
 	}
-#ifdef _WIN32
+#if defined(_WIN32) && !defined(MDB_VL32)
 	if (!(env->me_flags & MDB_RDONLY)) {
 		void *p;
 		p = (MDB_page *)(env->me_map + env->me_psize * pgno);

From 6f653ca205d7358be7c639b8711492560de3de2a Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 27 Jan 2016 11:48:22 +0000
Subject: [PATCH 218/504] MDB_VL32 more for
 1ba5adb2ec262405f9207d6015d4f29eea548d25

fix 32bit overflow in set_mapsize
---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index be36d4967e..41f7e25fcc 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -4272,7 +4272,7 @@ mdb_env_set_mapsize(MDB_env *env, mdb_size_t size)
 			size = meta->mm_mapsize;
 		{
 			/* Silently round up to minimum if the size is too small */
-			size_t minsize = (meta->mm_last_pg + 1) * env->me_psize;
+			mdb_size_t minsize = (meta->mm_last_pg + 1) * env->me_psize;
 			if (size < minsize)
 				size = minsize;
 		}

From 5bf313e820b8a6e122510e4571d9c6a3ec0136c1 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 28 Jan 2016 04:18:14 +0000
Subject: [PATCH 219/504] ITS#8363 Fix off-by-one in mdb_midl_shrink

---
 libraries/liblmdb/midl.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/midl.c b/libraries/liblmdb/midl.c
index e2005d16cf..152a1ec0dd 100644
--- a/libraries/liblmdb/midl.c
+++ b/libraries/liblmdb/midl.c
@@ -120,7 +120,7 @@ void mdb_midl_shrink( MDB_IDL *idp )
 {
 	MDB_IDL ids = *idp;
 	if (*(--ids) > MDB_IDL_UM_MAX &&
-		(ids = realloc(ids, (MDB_IDL_UM_MAX+1) * sizeof(MDB_ID))))
+		(ids = realloc(ids, (MDB_IDL_UM_MAX+2) * sizeof(MDB_ID))))
 	{
 		*ids++ = MDB_IDL_UM_MAX;
 		*idp = ids;

From 3f62ddc81c891f10c65b672e268ff2cb2b5fe228 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 28 Jan 2016 14:23:02 +0000
Subject: [PATCH 220/504] MDB_VL32 change overflow page scan

Just check the requested page, don't worry about any other pages
---
 libraries/liblmdb/mdb.c | 41 ++++++++++-------------------------------
 1 file changed, 10 insertions(+), 31 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 41f7e25fcc..986980a7a7 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5830,7 +5830,7 @@ retry:
 				if (rc)
 					goto fail;
 				if (!el[x].mref) {
-					munmap(el[x].mptr, el[x].mcnt);
+					munmap(el[x].mptr, env->me_psize * el[x].mcnt);
 					el[x].mptr = id3.mptr;
 					el[x].mcnt = id3.mcnt;
 				} else {
@@ -5882,36 +5882,15 @@ fail:
 			pthread_mutex_unlock(&env->me_rpmutex);
 			return rc;
 		}
-		/* If this page is far enough from the end of the env, scan for
-		 * any overflow pages that would spill onto another block.
-		 * Note we must compare against mt_last_pgno, the last written
-		 * page in the environment. Not mt_next_pgno, which increases
-		 * for every newly allocated (but not yet written) page. If
-		 * we scanned beyond the last written page we'd get a bus error.
-		 */
-		if (pgno + MDB_RPAGE_CHUNK <= txn->mt_last_pgno) {
-			int i;
-			char *cp = (char *)id3.mptr + rem * env->me_psize;
-			for (i=rem; i<MDB_RPAGE_CHUNK;) {
-				p = (MDB_page *)cp;
-				if (IS_OVERFLOW(p)) {
-					int nop = p->mp_pages;
-					if (nop + i > MDB_RPAGE_CHUNK) {
-						munmap(id3.mptr, len);
-						id3.mcnt = nop + i;
-						len = id3.mcnt * env->me_psize;
-						MAP(rc, env, id3.mptr, len, off);
-						if (rc)
-							goto fail;
-						break;
-					}
-					i += nop;
-					cp += nop * env->me_psize;
-				} else {
-					i++;
-					cp += env->me_psize;
-				}
-			}
+		/* check for overflow size */
+		p = (MDB_page *)((char *)id3.mptr + rem * env->me_psize);
+		if (IS_OVERFLOW(p) && p->mp_pages + rem > id3.mcnt) {
+			id3.mcnt = p->mp_pages + rem;
+			munmap(id3.mptr, len);
+			len = id3.mcnt * env->me_psize;
+			MAP(rc, env, id3.mptr, len, off);
+			if (rc)
+				goto fail;
 		}
 		mdb_mid3l_insert(el, &id3);
 		pthread_mutex_unlock(&env->me_rpmutex);

From e394e023e491ca0338ad1f485fd70a9dd3790ed2 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Thu, 28 Jan 2016 19:45:01 +0100
Subject: [PATCH 221/504] Fix MDB_VL32 mdb_cursor_count()/entrycount types

---
 libraries/liblmdb/lmdb.h | 2 +-
 libraries/liblmdb/mdb.c  | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 7bcdb9c704..ee375613f4 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -1544,7 +1544,7 @@ int  mdb_cursor_del(MDB_cursor *cursor, unsigned int flags);
 	 *	<li>EINVAL - cursor is not initialized, or an invalid parameter was specified.
 	 * </ul>
 	 */
-int  mdb_cursor_count(MDB_cursor *cursor, size_t *countp);
+int  mdb_cursor_count(MDB_cursor *cursor, mdb_size_t *countp);
 
 	/** @brief Compare two data items according to a particular database.
 	 *
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 986980a7a7..3552c73c60 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1075,7 +1075,7 @@ typedef struct MDB_db {
 	pgno_t		md_branch_pages;	/**< number of internal pages */
 	pgno_t		md_leaf_pages;		/**< number of leaf pages */
 	pgno_t		md_overflow_pages;	/**< number of overflow pages */
-	pgno_t		md_entries;		/**< number of data items */
+	mdb_size_t	md_entries;		/**< number of data items */
 	pgno_t		md_root;		/**< the root page of this tree */
 } MDB_db;
 
@@ -7545,7 +7545,7 @@ new_sub:
 		 */
 		if (do_sub) {
 			int xflags, new_dupdata;
-			size_t ecount;
+			mdb_size_t ecount;
 put_sub:
 			xdata.mv_size = 0;
 			xdata.mv_data = "";
@@ -8260,7 +8260,7 @@ mdb_cursor_renew(MDB_txn *txn, MDB_cursor *mc)
 
 /* Return the count of duplicate data items for the current key */
 int
-mdb_cursor_count(MDB_cursor *mc, size_t *countp)
+mdb_cursor_count(MDB_cursor *mc, mdb_size_t *countp)
 {
 	MDB_node	*leaf;
 

From 5f5f4dab9c4f265c46d345cf69da984cd58eba0e Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@symas.com>
Date: Sat, 30 Jan 2016 12:54:32 +0000
Subject: [PATCH 222/504] Happy New Year

---
 libraries/liblmdb/COPYRIGHT      | 2 +-
 libraries/liblmdb/intro.doc      | 2 +-
 libraries/liblmdb/lmdb.h         | 2 +-
 libraries/liblmdb/mdb.c          | 2 +-
 libraries/liblmdb/mdb_copy.1     | 2 +-
 libraries/liblmdb/mdb_copy.c     | 2 +-
 libraries/liblmdb/mdb_dump.1     | 2 +-
 libraries/liblmdb/mdb_dump.c     | 2 +-
 libraries/liblmdb/mdb_load.1     | 2 +-
 libraries/liblmdb/mdb_load.c     | 2 +-
 libraries/liblmdb/mdb_stat.1     | 2 +-
 libraries/liblmdb/mdb_stat.c     | 2 +-
 libraries/liblmdb/midl.c         | 2 +-
 libraries/liblmdb/midl.h         | 2 +-
 libraries/liblmdb/mtest.c        | 2 +-
 libraries/liblmdb/mtest2.c       | 2 +-
 libraries/liblmdb/mtest3.c       | 2 +-
 libraries/liblmdb/mtest4.c       | 2 +-
 libraries/liblmdb/mtest5.c       | 2 +-
 libraries/liblmdb/mtest6.c       | 2 +-
 libraries/liblmdb/sample-bdb.txt | 2 +-
 libraries/liblmdb/sample-mdb.txt | 2 +-
 22 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/libraries/liblmdb/COPYRIGHT b/libraries/liblmdb/COPYRIGHT
index 722d1a5154..fe4825af57 100644
--- a/libraries/liblmdb/COPYRIGHT
+++ b/libraries/liblmdb/COPYRIGHT
@@ -1,4 +1,4 @@
-Copyright 2011-2015 Howard Chu, Symas Corp.
+Copyright 2011-2016 Howard Chu, Symas Corp.
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/intro.doc b/libraries/liblmdb/intro.doc
index 870c7bb8e7..9fe9c22be1 100644
--- a/libraries/liblmdb/intro.doc
+++ b/libraries/liblmdb/intro.doc
@@ -1,5 +1,5 @@
 /*
- * Copyright 2015 Howard Chu, Symas Corp.
+ * Copyright 2015-2016 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index ee375613f4..a794c9f7d8 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -135,7 +135,7 @@
  *
  *	@author	Howard Chu, Symas Corporation.
  *
- *	@copyright Copyright 2011-2015 Howard Chu, Symas Corp. All rights reserved.
+ *	@copyright Copyright 2011-2016 Howard Chu, Symas Corp. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted only as authorized by the OpenLDAP
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 3552c73c60..47bcb3b422 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5,7 +5,7 @@
  *	BerkeleyDB API, but much simplified.
  */
 /*
- * Copyright 2011-2015 Howard Chu, Symas Corp.
+ * Copyright 2011-2016 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mdb_copy.1 b/libraries/liblmdb/mdb_copy.1
index 1e2a97694f..258affbfb4 100644
--- a/libraries/liblmdb/mdb_copy.1
+++ b/libraries/liblmdb/mdb_copy.1
@@ -1,5 +1,5 @@
 .TH MDB_COPY 1 "2014/06/20" "LMDB 0.9.14"
-.\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved.
+.\" Copyright 2012-2016 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_copy \- LMDB environment copy tool
diff --git a/libraries/liblmdb/mdb_copy.c b/libraries/liblmdb/mdb_copy.c
index f37ccbcc21..c953201097 100644
--- a/libraries/liblmdb/mdb_copy.c
+++ b/libraries/liblmdb/mdb_copy.c
@@ -1,6 +1,6 @@
 /* mdb_copy.c - memory-mapped database backup tool */
 /*
- * Copyright 2012-2015 Howard Chu, Symas Corp.
+ * Copyright 2012-2016 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mdb_dump.1 b/libraries/liblmdb/mdb_dump.1
index 5a647ba2ec..0b95ca175a 100644
--- a/libraries/liblmdb/mdb_dump.1
+++ b/libraries/liblmdb/mdb_dump.1
@@ -1,5 +1,5 @@
 .TH MDB_DUMP 1 "2014/06/20" "LMDB 0.9.14"
-.\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved.
+.\" Copyright 2014-2016 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_dump \- LMDB environment export tool
diff --git a/libraries/liblmdb/mdb_dump.c b/libraries/liblmdb/mdb_dump.c
index 72a469052d..5f3125fa2a 100644
--- a/libraries/liblmdb/mdb_dump.c
+++ b/libraries/liblmdb/mdb_dump.c
@@ -1,6 +1,6 @@
 /* mdb_dump.c - memory-mapped database dump tool */
 /*
- * Copyright 2011-2015 Howard Chu, Symas Corp.
+ * Copyright 2011-2016 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mdb_load.1 b/libraries/liblmdb/mdb_load.1
index 712ed0540d..90ea5b9983 100644
--- a/libraries/liblmdb/mdb_load.1
+++ b/libraries/liblmdb/mdb_load.1
@@ -1,5 +1,5 @@
 .TH MDB_LOAD 1 "2014/06/20" "LMDB 0.9.14"
-.\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved.
+.\" Copyright 2014-2016 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_load \- LMDB environment import tool
diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c
index d1fda4bf59..27aee40290 100644
--- a/libraries/liblmdb/mdb_load.c
+++ b/libraries/liblmdb/mdb_load.c
@@ -1,6 +1,6 @@
 /* mdb_load.c - memory-mapped database load tool */
 /*
- * Copyright 2011-2015 Howard Chu, Symas Corp.
+ * Copyright 2011-2016 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mdb_stat.1 b/libraries/liblmdb/mdb_stat.1
index 351c017578..b5ab6a9d23 100644
--- a/libraries/liblmdb/mdb_stat.1
+++ b/libraries/liblmdb/mdb_stat.1
@@ -1,5 +1,5 @@
 .TH MDB_STAT 1 "2014/06/20" "LMDB 0.9.14"
-.\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved.
+.\" Copyright 2012-2016 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_stat \- LMDB environment status tool
diff --git a/libraries/liblmdb/mdb_stat.c b/libraries/liblmdb/mdb_stat.c
index b785e7acf0..de5c3b514c 100644
--- a/libraries/liblmdb/mdb_stat.c
+++ b/libraries/liblmdb/mdb_stat.c
@@ -1,6 +1,6 @@
 /* mdb_stat.c - memory-mapped database status tool */
 /*
- * Copyright 2011-2015 Howard Chu, Symas Corp.
+ * Copyright 2011-2016 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/midl.c b/libraries/liblmdb/midl.c
index 152a1ec0dd..9748d8dba8 100644
--- a/libraries/liblmdb/midl.c
+++ b/libraries/liblmdb/midl.c
@@ -3,7 +3,7 @@
 /* $OpenLDAP$ */
 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
  *
- * Copyright 2000-2015 The OpenLDAP Foundation.
+ * Copyright 2000-2016 The OpenLDAP Foundation.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/midl.h b/libraries/liblmdb/midl.h
index ed1d75e36b..16a3522ae7 100644
--- a/libraries/liblmdb/midl.h
+++ b/libraries/liblmdb/midl.h
@@ -11,7 +11,7 @@
 /* $OpenLDAP$ */
 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
  *
- * Copyright 2000-2015 The OpenLDAP Foundation.
+ * Copyright 2000-2016 The OpenLDAP Foundation.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest.c b/libraries/liblmdb/mtest.c
index 9d15088b0c..28a33d4432 100644
--- a/libraries/liblmdb/mtest.c
+++ b/libraries/liblmdb/mtest.c
@@ -1,6 +1,6 @@
 /* mtest.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2015 Howard Chu, Symas Corp.
+ * Copyright 2011-2016 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest2.c b/libraries/liblmdb/mtest2.c
index eacbe59d53..b68aade819 100644
--- a/libraries/liblmdb/mtest2.c
+++ b/libraries/liblmdb/mtest2.c
@@ -1,6 +1,6 @@
 /* mtest2.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2015 Howard Chu, Symas Corp.
+ * Copyright 2011-2016 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest3.c b/libraries/liblmdb/mtest3.c
index 9db79e625d..73ee6e2144 100644
--- a/libraries/liblmdb/mtest3.c
+++ b/libraries/liblmdb/mtest3.c
@@ -1,6 +1,6 @@
 /* mtest3.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2015 Howard Chu, Symas Corp.
+ * Copyright 2011-2016 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest4.c b/libraries/liblmdb/mtest4.c
index 6df890e2d7..25666fffd3 100644
--- a/libraries/liblmdb/mtest4.c
+++ b/libraries/liblmdb/mtest4.c
@@ -1,6 +1,6 @@
 /* mtest4.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2015 Howard Chu, Symas Corp.
+ * Copyright 2011-2016 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest5.c b/libraries/liblmdb/mtest5.c
index 14e3c0da4a..d3d7b6286d 100644
--- a/libraries/liblmdb/mtest5.c
+++ b/libraries/liblmdb/mtest5.c
@@ -1,6 +1,6 @@
 /* mtest5.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2015 Howard Chu, Symas Corp.
+ * Copyright 2011-2016 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest6.c b/libraries/liblmdb/mtest6.c
index ae3c7f264c..1b47db83f5 100644
--- a/libraries/liblmdb/mtest6.c
+++ b/libraries/liblmdb/mtest6.c
@@ -1,6 +1,6 @@
 /* mtest6.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2015 Howard Chu, Symas Corp.
+ * Copyright 2011-2016 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/sample-bdb.txt b/libraries/liblmdb/sample-bdb.txt
index 563807a2b0..11aff13f2e 100644
--- a/libraries/liblmdb/sample-bdb.txt
+++ b/libraries/liblmdb/sample-bdb.txt
@@ -3,7 +3,7 @@
  * Do a line-by-line comparison of this and sample-mdb.txt
  */
 /*
- * Copyright 2012-2015 Howard Chu, Symas Corp.
+ * Copyright 2012-2016 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/sample-mdb.txt b/libraries/liblmdb/sample-mdb.txt
index 10a256870b..5d3373736d 100644
--- a/libraries/liblmdb/sample-mdb.txt
+++ b/libraries/liblmdb/sample-mdb.txt
@@ -3,7 +3,7 @@
  * Do a line-by-line comparison of this and sample-bdb.txt
  */
 /*
- * Copyright 2012-2015 Howard Chu, Symas Corp.
+ * Copyright 2012-2016 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without

From fcac8d07742891cedcf94b982e5094315132f62a Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Thu, 4 Feb 2016 03:23:13 +0100
Subject: [PATCH 223/504] ITS#7992 Fix memleak in prev change

---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 47bcb3b422..ea9e472902 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -10120,7 +10120,7 @@ mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags)
 #ifdef _WIN32
 	rc = utf8_to_utf16(lpath, -1, &wpath, NULL);
 	if (rc)
-		return rc;
+		goto leave;
 	newfd = CreateFileW(wpath, GENERIC_WRITE, 0, NULL, CREATE_NEW,
 				FILE_FLAG_NO_BUFFERING|FILE_FLAG_WRITE_THROUGH, NULL);
 	free(wpath);

From 5ef1908224121c01eb9190b03978d563cc0c22d2 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sun, 14 Feb 2016 23:53:05 +0000
Subject: [PATCH 224/504] ITS#8324 Map NTAPI result codes to WIN32 codes

---
 libraries/liblmdb/mdb.c | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index ea9e472902..cddc746043 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -4159,6 +4159,19 @@ mdb_env_create(MDB_env **env)
 	return MDB_SUCCESS;
 }
 
+#ifdef _WIN32
+/** @brief Map a result from an NTAPI call to WIN32. */
+static DWORD
+mdb_nt2win32(NTSTATUS st)
+{
+	OVERLAPPED o = {0};
+	DWORD br;
+	o.Internal = st;
+	GetOverlappedResult(NULL, &o, &br, FALSE);
+	return GetLastError();
+}
+#endif
+
 static int ESECT
 mdb_env_map(MDB_env *env, void *addr)
 {
@@ -4188,7 +4201,7 @@ mdb_env_map(MDB_env *env, void *addr)
 
 	rc = NtCreateSection(&mh, access, NULL, NULL, secprot, SEC_RESERVE, env->me_fd);
 	if (rc)
-		return rc;
+		return mdb_nt2win32(rc);
 	map = addr;
 #ifdef MDB_VL32
 	msize = NUM_METAS * env->me_psize;
@@ -4200,7 +4213,7 @@ mdb_env_map(MDB_env *env, void *addr)
 	NtClose(mh);
 #endif
 	if (rc)
-		return rc;
+		return mdb_nt2win32(rc);
 	env->me_map = map;
 #else
 #ifdef MDB_VL32
@@ -5692,7 +5705,8 @@ mdb_rpage_get(MDB_txn *txn, pgno_t pg0, MDB_page **ret)
 #define MAP(rc,env,addr,len,off)	\
 	addr = NULL; \
 	rc = NtMapViewOfSection(env->me_fmh, GetCurrentProcess(), &addr, 0, \
-		len, &off, &len, ViewUnmap, (env->me_flags & MDB_RDONLY) ? 0 : MEM_RESERVE, PAGE_READONLY)
+		len, &off, &len, ViewUnmap, (env->me_flags & MDB_RDONLY) ? 0 : MEM_RESERVE, PAGE_READONLY); \
+	if (rc) rc = mdb_nt2win32(rc)
 #else
 	off_t off;
 	size_t len;

From d909ab2f36e079087047dd842210b37cec8b6a84 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 15 Feb 2016 00:07:04 +0000
Subject: [PATCH 225/504] Tweak Win32 errmsg buffer

---
 libraries/liblmdb/mdb.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index cddc746043..f5980807f6 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1572,8 +1572,9 @@ mdb_strerror(int err)
 	 *	This works as long as no function between the call to mdb_strerror
 	 *	and the actual use of the message uses more than 4K of stack.
 	 */
-	char pad[4096];
-	char buf[1024], *ptr = buf;
+#define MSGSIZE	1024
+#define PADSIZE 4096
+	char buf[MSGSIZE+PADSIZE], *ptr = buf;
 #endif
 	int i;
 	if (!err)
@@ -1605,7 +1606,7 @@ mdb_strerror(int err)
 	buf[0] = 0;
 	FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM |
 		FORMAT_MESSAGE_IGNORE_INSERTS,
-		NULL, err, 0, ptr, sizeof(buf), (va_list *)pad);
+		NULL, err, 0, ptr, MSGSIZE, (va_list *)buf+MSGSIZE);
 	return ptr;
 #else
 	return strerror(err);

From 3f62b727ccf3424daca1cdc24bbf98c869f44699 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 16 Feb 2016 23:34:27 +0000
Subject: [PATCH 226/504] Tweak MDB_PREV_MULTIPLE for uninit'd cursor

---
 libraries/liblmdb/mdb.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index f5980807f6..a366b40245 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1573,7 +1573,7 @@ mdb_strerror(int err)
 	 *	and the actual use of the message uses more than 4K of stack.
 	 */
 #define MSGSIZE	1024
-#define PADSIZE 4096
+#define PADSIZE	4096
 	char buf[MSGSIZE+PADSIZE], *ptr = buf;
 #endif
 	int i;
@@ -7007,8 +7007,10 @@ fetchm:
 			break;
 		}
 		if (!(mc->mc_flags & C_INITIALIZED))
-			rc = mdb_cursor_first(mc, key, data);
-		else {
+			rc = mdb_cursor_last(mc, key, data);
+		else
+			rc = MDB_SUCCESS;
+		if (rc == MDB_SUCCESS) {
 			MDB_cursor *mx = &mc->mc_xcursor->mx_cursor;
 			if (mx->mc_flags & C_INITIALIZED) {
 				rc = mdb_cursor_sibling(mx, 0);

From e46d78b7b083c3a9983f8a0e77b90bccd64b6ec9 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@symas.com>
Date: Wed, 17 Feb 2016 17:27:50 +0000
Subject: [PATCH 227/504] MDB_VL32 - increase max write txn size

---
 libraries/liblmdb/midl.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/midl.h b/libraries/liblmdb/midl.h
index 16a3522ae7..b0d518ef03 100644
--- a/libraries/liblmdb/midl.h
+++ b/libraries/liblmdb/midl.h
@@ -61,7 +61,7 @@ typedef MDB_ID *MDB_IDL;
  *   limiting factors: sizeof(ID), thread stack size
  */
 #ifdef MDB_VL32
-#define	MDB_IDL_LOGN	10	/* DB_SIZE is 2^10, UM_SIZE is 2^11 */
+#define	MDB_IDL_LOGN	14	/* DB_SIZE is 2^14, UM_SIZE is 2^15 */
 #else
 #define	MDB_IDL_LOGN	16	/* DB_SIZE is 2^16, UM_SIZE is 2^17 */
 #endif

From 8fff90db7ed1e650395f39f9e8b22673aa74aa2b Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 28 Mar 2016 01:35:56 +0100
Subject: [PATCH 228/504] ITS#8393 fix MDB_GET_BOTH on non-dup record

---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index a366b40245..acab07b829 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -6771,8 +6771,8 @@ set1:
 				if (op == MDB_GET_BOTH || rc > 0)
 					return MDB_NOTFOUND;
 				rc = 0;
-				*data = olddata;
 			}
+			*data = olddata;
 
 		} else {
 			if (mc->mc_xcursor)

From c8dbd772f751471fd2824bbb27ac9b8b392af465 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sat, 9 Apr 2016 20:42:45 +0100
Subject: [PATCH 229/504] mdb_drop optimization

If we know there are no sub-DBs and no overflow pages, skip leaf scan.
---
 libraries/liblmdb/mdb.c | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index acab07b829..2d8e458f7e 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -10501,8 +10501,11 @@ mdb_drop0(MDB_cursor *mc, int subs)
 
 		/* DUPSORT sub-DBs have no ovpages/DBs. Omit scanning leaves.
 		 * This also avoids any P_LEAF2 pages, which have no nodes.
+		 * Also if the DB doesn't have sub-DBs and has no overflow
+		 * pages, omit scanning leaves.
 		 */
-		if (mc->mc_flags & C_SUB)
+		if ((mc->mc_flags & C_SUB) ||
+			(!subs && !mc->mc_db->md_overflow_pages))
 			mdb_cursor_pop(mc);
 
 		mdb_cursor_copy(mc, &mx);
@@ -10529,6 +10532,9 @@ mdb_drop0(MDB_cursor *mc, int subs)
 							pg, omp->mp_pages);
 						if (rc)
 							goto done;
+						mc->mc_db->md_overflow_pages -= omp->mp_pages;
+						if (!mc->mc_db->md_overflow_pages && !subs)
+							break;
 					} else if (subs && (ni->mn_flags & F_SUBDATA)) {
 						mdb_xcursor_init1(mc, ni);
 						rc = mdb_drop0(&mc->mc_xcursor->mx_cursor, 0);
@@ -10536,6 +10542,8 @@ mdb_drop0(MDB_cursor *mc, int subs)
 							goto done;
 					}
 				}
+				if (!subs && !mc->mc_db->md_overflow_pages)
+					goto pop;
 			} else {
 				if ((rc = mdb_midl_need(&txn->mt_free_pgs, n)) != 0)
 					goto done;
@@ -10557,6 +10565,7 @@ mdb_drop0(MDB_cursor *mc, int subs)
 				/* no more siblings, go back to beginning
 				 * of previous level.
 				 */
+pop:
 				mdb_cursor_pop(mc);
 				mc->mc_ki[0] = 0;
 				for (i=1; i<mc->mc_snum; i++) {

From 37081325f7356587c5e6ce4c1f36c3b303fa718c Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 18 Apr 2016 18:07:56 +0100
Subject: [PATCH 230/504] ITS#8406 fix xcursors after cursor_del

Don't leave them uninit'd if they now point at a valid DUP node
---
 libraries/liblmdb/mdb.c | 50 ++++++++++++++++++++---------------------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 2d8e458f7e..e62ca628e7 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -6420,8 +6420,8 @@ mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
 	if (mc->mc_flags & C_EOF) {
 		return MDB_NOTFOUND;
 	}
-
-	mdb_cassert(mc, mc->mc_flags & C_INITIALIZED);
+	if (!(mc->mc_flags & C_INITIALIZED))
+		return mdb_cursor_first(mc, key, data);
 
 	mp = mc->mc_pg[mc->mc_top];
 
@@ -6507,7 +6507,12 @@ mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
 	MDB_node	*leaf;
 	int rc;
 
-	mdb_cassert(mc, mc->mc_flags & C_INITIALIZED);
+	if (!(mc->mc_flags & C_INITIALIZED)) {
+		rc = mdb_cursor_last(mc, key, data);
+		if (rc)
+			return rc;
+		mc->mc_ki[mc->mc_top]++;
+	}
 
 	mp = mc->mc_pg[mc->mc_top];
 
@@ -6979,10 +6984,7 @@ mdb_cursor_get(MDB_cursor *mc, MDB_val *key, MDB_val *data,
 			rc = MDB_INCOMPATIBLE;
 			break;
 		}
-		if (!(mc->mc_flags & C_INITIALIZED))
-			rc = mdb_cursor_first(mc, key, data);
-		else
-			rc = mdb_cursor_next(mc, key, data, MDB_NEXT_DUP);
+		rc = mdb_cursor_next(mc, key, data, MDB_NEXT_DUP);
 		if (rc == MDB_SUCCESS) {
 			if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) {
 				MDB_cursor *mx;
@@ -7024,21 +7026,11 @@ fetchm:
 	case MDB_NEXT:
 	case MDB_NEXT_DUP:
 	case MDB_NEXT_NODUP:
-		if (!(mc->mc_flags & C_INITIALIZED))
-			rc = mdb_cursor_first(mc, key, data);
-		else
-			rc = mdb_cursor_next(mc, key, data, op);
+		rc = mdb_cursor_next(mc, key, data, op);
 		break;
 	case MDB_PREV:
 	case MDB_PREV_DUP:
 	case MDB_PREV_NODUP:
-		if (!(mc->mc_flags & C_INITIALIZED)) {
-			rc = mdb_cursor_last(mc, key, data);
-			if (rc)
-				break;
-			mc->mc_flags |= C_INITIALIZED;
-			mc->mc_ki[mc->mc_top]++;
-		}
 		rc = mdb_cursor_prev(mc, key, data, op);
 		break;
 	case MDB_FIRST:
@@ -9050,8 +9042,6 @@ mdb_cursor_del0(MDB_cursor *mc)
 			if (m3->mc_pg[mc->mc_top] == mp) {
 				if (m3->mc_ki[mc->mc_top] == ki) {
 					m3->mc_flags |= C_DEL;
-					if (mc->mc_db->md_flags & MDB_DUPSORT)
-						m3->mc_xcursor->mx_cursor.mc_flags &= ~C_INITIALIZED;
 				} else if (m3->mc_ki[mc->mc_top] > ki) {
 					m3->mc_ki[mc->mc_top]--;
 				}
@@ -9085,11 +9075,21 @@ mdb_cursor_del0(MDB_cursor *mc)
 				continue;
 			if (m3->mc_pg[mc->mc_top] == mp) {
 				/* if m3 points past last node in page, find next sibling */
-				if (m3->mc_ki[mc->mc_top] >= nkeys) {
-					rc = mdb_cursor_sibling(m3, 1);
-					if (rc == MDB_NOTFOUND) {
-						m3->mc_flags |= C_EOF;
-						rc = MDB_SUCCESS;
+				if (m3->mc_ki[mc->mc_top] >= mc->mc_ki[mc->mc_top]) {
+					if (m3->mc_ki[mc->mc_top] >= nkeys) {
+						rc = mdb_cursor_sibling(m3, 1);
+						if (rc == MDB_NOTFOUND) {
+							m3->mc_flags |= C_EOF;
+							rc = MDB_SUCCESS;
+							continue;
+						}
+					}
+					if (mc->mc_db->md_flags & MDB_DUPSORT) {
+						MDB_node *node = NODEPTR(m3->mc_pg[m3->mc_top], m3->mc_ki[m3->mc_top]);
+						if (node->mn_flags & F_DUPDATA) {
+							mdb_xcursor_init1(m3, node);
+							m3->mc_xcursor->mx_cursor.mc_flags |= C_DEL;
+						}
 					}
 				}
 			}

From a04aad31c248401694d49d4c041db26d8b7ac310 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 26 Apr 2016 12:52:21 +0100
Subject: [PATCH 231/504] ITS#8412 fix NEXT_DUP after cursor_del

---
 libraries/liblmdb/mdb.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index e62ca628e7..72294a7e51 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -6417,7 +6417,8 @@ mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
 	MDB_node	*leaf;
 	int rc;
 
-	if (mc->mc_flags & C_EOF) {
+	if ((mc->mc_flags & C_EOF) ||
+		((mc->mc_flags & C_DEL) && op == MDB_NEXT_DUP)) {
 		return MDB_NOTFOUND;
 	}
 	if (!(mc->mc_flags & C_INITIALIZED))

From e2b8b6448de7de5229a44e18c224e3a670da4bdc Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 10 May 2016 07:11:44 +0200
Subject: [PATCH 232/504] Comment ovpage code in mdb_cursor_put()

---
 libraries/liblmdb/mdb.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 72294a7e51..f3ca42755c 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7468,8 +7468,13 @@ current:
 					/* Note - this page is already counted in parent's dirty_room */
 					rc2 = mdb_mid2l_insert(mc->mc_txn->mt_u.dirty_list, &id2);
 					mdb_cassert(mc, rc2 == 0);
+					/* Currently we make the page look as with put() in the
+					 * parent txn, in case the user peeks at MDB_RESERVEd
+					 * or unused parts. Some users treat ovpages specially.
+					 */
 					if (!(flags & MDB_RESERVE)) {
-						/* Copy end of page, adjusting alignment so
+						/* Skip the part where LMDB will put *data.
+						 * Copy end of page, adjusting alignment so
 						 * compiler may copy words instead of bytes.
 						 */
 						off = (PAGEHDRSZ + data->mv_size) & -sizeof(size_t);

From b045bce26037213f4f591787ac615a0463b031a8 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sun, 15 May 2016 00:44:54 +0100
Subject: [PATCH 233/504] ITS#8424 init cursor in mdb_env_cwalk

---
 libraries/liblmdb/mdb.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index f3ca42755c..af4d73a73a 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -9732,7 +9732,7 @@ mdb_env_cthr_toggle(mdb_copy *my, int st)
 static int ESECT
 mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags)
 {
-	MDB_cursor mc;
+	MDB_cursor mc = {0};
 	MDB_node *ni;
 	MDB_page *mo, *mp, *leaf;
 	char *buf, *ptr;
@@ -9744,8 +9744,8 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags)
 		return MDB_SUCCESS;
 
 	mc.mc_snum = 1;
-	mc.mc_top = 0;
 	mc.mc_txn = my->mc_txn;
+	mc.mc_flags = my->mc_txn->mt_flags & (C_ORIG_RDONLY|C_WRITEMAP);
 
 	rc = mdb_page_get(&mc, *pg, &mc.mc_pg[0], NULL);
 	if (rc)

From c367c1f69685a4d307acb8cea6945c1d67e1cc7e Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 2 Jun 2016 21:01:27 +0100
Subject: [PATCH 234/504] ITS#8339 Solaris 10/11 robust mutex fixes

Check for PTHREAD_MUTEX_ROBUST_NP definition (this doesn't work
on Linux/glibc because they used an enum). Zero out mutex before
initing.
---
 libraries/liblmdb/mdb.c | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index af4d73a73a..79a958b0ff 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -304,7 +304,8 @@ union semun {
 # else
 #  define MDB_USE_ROBUST	1
 /* glibc < 2.12 only provided _np API */
-#  if defined(__GLIBC__) && GLIBC_VER < 0x02000c
+#  if (defined(__GLIBC__) && GLIBC_VER < 0x02000c) || \
+	(defined(PTHREAD_MUTEX_ROBUST_NP) && !defined(PTHREAD_MUTEX_ROBUST))
 #   define PTHREAD_MUTEX_ROBUST	PTHREAD_MUTEX_ROBUST_NP
 #   define pthread_mutexattr_setrobust(attr, flag)	pthread_mutexattr_setrobust_np(attr, flag)
 #   define pthread_mutex_consistent(mutex)	pthread_mutex_consistent_np(mutex)
@@ -4962,6 +4963,13 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 #else	/* MDB_USE_POSIX_MUTEX: */
 		pthread_mutexattr_t mattr;
 
+		/* Solaris needs this before initing a robust mutex.  Otherwise
+		 * it may skip the init and return EBUSY "seems someone already
+		 * inited" or EINVAL "it was inited differently".
+		 */
+		memset(env->me_txns->mti_rmutex, 0, sizeof(*env->me_txns->mti_rmutex));
+		memset(env->me_txns->mti_wmutex, 0, sizeof(*env->me_txns->mti_wmutex));
+
 		if ((rc = pthread_mutexattr_init(&mattr))
 			|| (rc = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED))
 #ifdef MDB_ROBUST_SUPPORTED

From 53a0fdf1bee795048b3b3a40f8ffec3e3f8473a2 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Fri, 3 Jun 2016 06:11:54 +0200
Subject: [PATCH 235/504] Init "locked" flag for SysV semaphores

---
 libraries/liblmdb/mdb.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 79a958b0ff..29ca141c45 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -4960,6 +4960,8 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 		if (semctl(semid, 0, SETALL, semu) < 0)
 			goto fail_errno;
 		env->me_txns->mti_semid = semid;
+		env->me_txns->mti_rlocked = 0;
+		env->me_txns->mti_wlocked = 0;
 #else	/* MDB_USE_POSIX_MUTEX: */
 		pthread_mutexattr_t mattr;
 

From c4c7833d245fdc4b2ea4a1a459d7069ce3af87f5 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sun, 5 Jun 2016 23:42:44 +0200
Subject: [PATCH 236/504] mdb_env_setup_locks: Plug mutexattr leak on error

---
 libraries/liblmdb/mdb.c | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 29ca141c45..0c9effee46 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -4972,15 +4972,17 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 		memset(env->me_txns->mti_rmutex, 0, sizeof(*env->me_txns->mti_rmutex));
 		memset(env->me_txns->mti_wmutex, 0, sizeof(*env->me_txns->mti_wmutex));
 
-		if ((rc = pthread_mutexattr_init(&mattr))
-			|| (rc = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED))
-#ifdef MDB_ROBUST_SUPPORTED
-			|| (rc = pthread_mutexattr_setrobust(&mattr, PTHREAD_MUTEX_ROBUST))
-#endif
-			|| (rc = pthread_mutex_init(env->me_txns->mti_rmutex, &mattr))
-			|| (rc = pthread_mutex_init(env->me_txns->mti_wmutex, &mattr)))
+		if ((rc = pthread_mutexattr_init(&mattr)) != 0)
 			goto fail;
+		rc = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED);
+#ifdef MDB_ROBUST_SUPPORTED
+		if (!rc) rc = pthread_mutexattr_setrobust(&mattr, PTHREAD_MUTEX_ROBUST);
+#endif
+		if (!rc) rc = pthread_mutex_init(env->me_txns->mti_rmutex, &mattr);
+		if (!rc) rc = pthread_mutex_init(env->me_txns->mti_wmutex, &mattr);
 		pthread_mutexattr_destroy(&mattr);
+		if (rc)
+			goto fail;
 #endif	/* _WIN32 || ... */
 
 		env->me_txns->mti_magic = MDB_MAGIC;

From eb7bbed967cd2a54d90d997e3470d02262baadb2 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 25 Jun 2016 07:55:34 +0200
Subject: [PATCH 237/504] ITS#8209 MDB_CP_COMPACT fixes

Handle errors.  Fix cond_wait condition so mc_new
is the sole control var.  Drop specious cond_waits.
Do not look at 'mo' while copythr writes it.
---
 libraries/liblmdb/mdb.c | 133 ++++++++++++++++++++--------------------
 1 file changed, 67 insertions(+), 66 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 0c9effee46..77b05b1edc 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -336,8 +336,10 @@ typedef HANDLE mdb_mutex_t, mdb_mutexref_t;
 #define pthread_mutex_lock(x)	WaitForSingleObject(*x, INFINITE)
 #define pthread_cond_signal(x)	SetEvent(*x)
 #define pthread_cond_wait(cond,mutex)	do{SignalObjectAndWait(*mutex, *cond, INFINITE, FALSE); WaitForSingleObject(*mutex, INFINITE);}while(0)
-#define THREAD_CREATE(thr,start,arg)	thr=CreateThread(NULL,0,start,arg,0,NULL)
-#define THREAD_FINISH(thr)	WaitForSingleObject(thr, INFINITE)
+#define THREAD_CREATE(thr,start,arg) \
+	(((thr) = CreateThread(NULL, 0, start, arg, 0, NULL)) ? 0 : ErrCode())
+#define THREAD_FINISH(thr) \
+	(WaitForSingleObject(thr, INFINITE) ? ErrCode() : 0)
 #define LOCK_MUTEX0(mutex)		WaitForSingleObject(mutex, INFINITE)
 #define UNLOCK_MUTEX(mutex)		ReleaseMutex(mutex)
 #define mdb_mutex_consistent(mutex)	0
@@ -9637,11 +9639,12 @@ mdb_put(MDB_txn *txn, MDB_dbi dbi,
 #ifndef MDB_WBUF
 #define MDB_WBUF	(1024*1024)
 #endif
+#define MDB_EOF		0x10	/**< #mdb_env_copyfd1() is done reading */
 
-	/** State needed for a compacting copy. */
+	/** State needed for a double-buffering compacting copy. */
 typedef struct mdb_copy {
 	pthread_mutex_t mc_mutex;
-	pthread_cond_t mc_cond;
+	pthread_cond_t mc_cond;	/**< Condition variable for #mc_new */
 	char *mc_wbuf[2];
 	char *mc_over[2];
 	MDB_env *mc_env;
@@ -9650,10 +9653,9 @@ typedef struct mdb_copy {
 	int mc_olen[2];
 	pgno_t mc_next_pgno;
 	HANDLE mc_fd;
-	int mc_status;
-	volatile int mc_new;
-	int mc_toggle;
-
+	int mc_toggle;			/**< Buffer number in provider */
+	int mc_new;				/**< (0-2 buffers to write) | (#MDB_EOF at end) */
+	volatile int mc_error;	/**< Error code, never cleared if set */
 } mdb_copy;
 
 	/** Dedicated writer thread for compacting copy. */
@@ -9672,20 +9674,16 @@ mdb_env_copythr(void *arg)
 #endif
 
 	pthread_mutex_lock(&my->mc_mutex);
-	my->mc_new = 0;
-	pthread_cond_signal(&my->mc_cond);
 	for(;;) {
 		while (!my->mc_new)
 			pthread_cond_wait(&my->mc_cond, &my->mc_mutex);
-		if (my->mc_new < 0) {
-			my->mc_new = 0;
+		if (my->mc_new == 0 + MDB_EOF) /* 0 buffers, just EOF */
 			break;
-		}
-		my->mc_new = 0;
 		wsize = my->mc_wlen[toggle];
 		ptr = my->mc_wbuf[toggle];
 again:
-		while (wsize > 0) {
+		rc = MDB_SUCCESS;
+		while (wsize > 0 && !my->mc_error) {
 			DO_WRITE(rc, my->mc_fd, ptr, wsize, len);
 			if (!rc) {
 				rc = ErrCode();
@@ -9701,8 +9699,7 @@ again:
 			}
 		}
 		if (rc) {
-			my->mc_status = rc;
-			break;
+			my->mc_error = rc;
 		}
 		/* If there's an overflow page tail, write it too */
 		if (my->mc_olen[toggle]) {
@@ -9713,31 +9710,34 @@ again:
 		}
 		my->mc_wlen[toggle] = 0;
 		toggle ^= 1;
+		/* Return the empty buffer to provider */
+		my->mc_new--;
 		pthread_cond_signal(&my->mc_cond);
 	}
-	pthread_cond_signal(&my->mc_cond);
 	pthread_mutex_unlock(&my->mc_mutex);
 	return (THREAD_RET)0;
 #undef DO_WRITE
 }
 
-	/** Tell the writer thread there's a buffer ready to write */
+	/** Give buffer and/or #MDB_EOF to writer thread, await unused buffer.
+	 *
+	 * @param[in] my control structure.
+	 * @param[in] adjust (1 to hand off 1 buffer) | (MDB_EOF when ending).
+	 */
 static int ESECT
-mdb_env_cthr_toggle(mdb_copy *my, int st)
+mdb_env_cthr_toggle(mdb_copy *my, int adjust)
 {
-	int toggle = my->mc_toggle ^ 1;
 	pthread_mutex_lock(&my->mc_mutex);
-	if (my->mc_status) {
-		pthread_mutex_unlock(&my->mc_mutex);
-		return my->mc_status;
-	}
-	while (my->mc_new == 1)
-		pthread_cond_wait(&my->mc_cond, &my->mc_mutex);
-	my->mc_new = st;
-	my->mc_toggle = toggle;
+	my->mc_new += adjust;
 	pthread_cond_signal(&my->mc_cond);
+	while (my->mc_new & 2)		/* both buffers in use */
+		pthread_cond_wait(&my->mc_cond, &my->mc_mutex);
 	pthread_mutex_unlock(&my->mc_mutex);
-	return 0;
+
+	my->mc_toggle ^= (adjust & 1);
+	/* Both threads reset mc_wlen, to be safe from threading errors */
+	my->mc_wlen[my->mc_toggle] = 0;
+	return my->mc_error;
 }
 
 	/** Depth-first tree traversal for compacting copy. */
@@ -9803,6 +9803,7 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags)
 						}
 
 						memcpy(&pg, NODEDATA(ni), sizeof(pg));
+						memcpy(NODEDATA(ni), &my->mc_next_pgno, sizeof(pgno_t));
 						rc = mdb_page_get(&mc, pg, &omp, NULL);
 						if (rc)
 							goto done;
@@ -9825,7 +9826,6 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags)
 								goto done;
 							toggle = my->mc_toggle;
 						}
-						memcpy(NODEDATA(ni), &mo->mp_pgno, sizeof(pgno_t));
 					} else if (ni->mn_flags & F_SUBDATA) {
 						MDB_db db;
 
@@ -9903,47 +9903,51 @@ mdb_env_copyfd1(MDB_env *env, HANDLE fd)
 {
 	MDB_meta *mm;
 	MDB_page *mp;
-	mdb_copy my;
+	mdb_copy my = {0};
 	MDB_txn *txn = NULL;
 	pthread_t thr;
-	int rc;
+	int rc = MDB_SUCCESS;
 
 #ifdef _WIN32
-	my.mc_mutex = CreateMutex(NULL, FALSE, NULL);
-	my.mc_cond = CreateEvent(NULL, FALSE, FALSE, NULL);
+	if (!(my.mc_mutex = CreateMutex(NULL, FALSE, NULL)) ||
+		!(my.mc_cond = CreateEvent(NULL, FALSE, FALSE, NULL))) {
+		rc = ErrCode();
+		goto done;
+	}
 	my.mc_wbuf[0] = _aligned_malloc(MDB_WBUF*2, env->me_os_psize);
-	if (my.mc_wbuf[0] == NULL)
-		return errno;
+	if (my.mc_wbuf[0] == NULL) {
+		/* _aligned_malloc() sets errno, but we use Windows error codes */
+		rc = ERROR_NOT_ENOUGH_MEMORY;
+		goto done;
+	}
 #else
-	pthread_mutex_init(&my.mc_mutex, NULL);
-	pthread_cond_init(&my.mc_cond, NULL);
+	if ((rc = pthread_mutex_init(&my.mc_mutex, NULL)) ||
+		(rc = pthread_cond_init(&my.mc_cond, NULL)))
+		return rc;
 #ifdef HAVE_MEMALIGN
 	my.mc_wbuf[0] = memalign(env->me_os_psize, MDB_WBUF*2);
-	if (my.mc_wbuf[0] == NULL)
-		return errno;
+	if (my.mc_wbuf[0] == NULL) {
+		rc = errno;
+		goto done;
+	}
 #else
 	rc = posix_memalign((void **)&my.mc_wbuf[0], env->me_os_psize, MDB_WBUF*2);
 	if (rc)
-		return rc;
+		goto done;
 #endif
 #endif
 	memset(my.mc_wbuf[0], 0, MDB_WBUF*2);
 	my.mc_wbuf[1] = my.mc_wbuf[0] + MDB_WBUF;
-	my.mc_wlen[0] = 0;
-	my.mc_wlen[1] = 0;
-	my.mc_olen[0] = 0;
-	my.mc_olen[1] = 0;
 	my.mc_next_pgno = NUM_METAS;
-	my.mc_status = 0;
-	my.mc_new = 1;
-	my.mc_toggle = 0;
 	my.mc_env = env;
 	my.mc_fd = fd;
-	THREAD_CREATE(thr, mdb_env_copythr, &my);
+	rc = THREAD_CREATE(thr, mdb_env_copythr, &my);
+	if (rc)
+		goto done;
 
 	rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
 	if (rc)
-		return rc;
+		goto finish;
 
 	mp = (MDB_page *)my.mc_wbuf[0];
 	memset(mp, 0, NUM_METAS * env->me_psize);
@@ -9969,6 +9973,8 @@ mdb_env_copyfd1(MDB_env *env, HANDLE fd)
 		mdb_cursor_init(&mc, txn, FREE_DBI, NULL);
 		while ((rc = mdb_cursor_get(&mc, &key, &data, MDB_NEXT)) == 0)
 			freecount += *(MDB_ID *)data.mv_data;
+		if (rc != MDB_NOTFOUND)
+			goto finish;
 		freecount += txn->mt_dbs[FREE_DBI].md_branch_pages +
 			txn->mt_dbs[FREE_DBI].md_leaf_pages +
 			txn->mt_dbs[FREE_DBI].md_overflow_pages;
@@ -9985,31 +9991,26 @@ mdb_env_copyfd1(MDB_env *env, HANDLE fd)
 	}
 	my.mc_wlen[0] = env->me_psize * NUM_METAS;
 	my.mc_txn = txn;
-	pthread_mutex_lock(&my.mc_mutex);
-	while(my.mc_new)
-		pthread_cond_wait(&my.mc_cond, &my.mc_mutex);
-	pthread_mutex_unlock(&my.mc_mutex);
 	rc = mdb_env_cwalk(&my, &txn->mt_dbs[MAIN_DBI].md_root, 0);
-	if (rc == MDB_SUCCESS && my.mc_wlen[my.mc_toggle])
-		rc = mdb_env_cthr_toggle(&my, 1);
-	mdb_env_cthr_toggle(&my, -1);
-	pthread_mutex_lock(&my.mc_mutex);
-	while(my.mc_new)
-		pthread_cond_wait(&my.mc_cond, &my.mc_mutex);
-	pthread_mutex_unlock(&my.mc_mutex);
-	THREAD_FINISH(thr);
 
+finish:
+	if (rc)
+		my.mc_error = rc;
+	mdb_env_cthr_toggle(&my, 1 | MDB_EOF);
+	rc = THREAD_FINISH(thr);
 	mdb_txn_abort(txn);
+
+done:
 #ifdef _WIN32
-	CloseHandle(my.mc_cond);
-	CloseHandle(my.mc_mutex);
+	if (my.mc_cond)  CloseHandle(my.mc_cond);
+	if (my.mc_mutex) CloseHandle(my.mc_mutex);
 	_aligned_free(my.mc_wbuf[0]);
 #else
 	pthread_cond_destroy(&my.mc_cond);
 	pthread_mutex_destroy(&my.mc_mutex);
 	free(my.mc_wbuf[0]);
 #endif
-	return rc;
+	return rc ? rc : my.mc_error;
 }
 
 	/** Copy environment as-is. */

From 5ea12b0be85ffe3be49692af90e24411c5b90655 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 25 Jun 2016 07:57:04 +0200
Subject: [PATCH 238/504] ITS#8209 MDB_CP_COMPACT: Handle empty or broken DB

Preserve DB flags (use metapage#1) when main DB is empty.
Fail if metapage root != actual root in output file.
---
 libraries/liblmdb/lmdb.h     |  1 +
 libraries/liblmdb/mdb.c      | 36 +++++++++++++++++++++++-------------
 libraries/liblmdb/mdb_copy.1 |  1 +
 3 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index a794c9f7d8..85dc50d580 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -688,6 +688,7 @@ int  mdb_env_copyfd(MDB_env *env, mdb_filehandle_t fd);
 	 *	<li>#MDB_CP_COMPACT - Perform compaction while copying: omit free
 	 *		pages and sequentially renumber all pages in output. This option
 	 *		consumes more CPU and runs more slowly than the default.
+	 *		Currently it fails if the environment has suffered a page leak.
 	 * </ul>
 	 * @return A non-zero error value on failure and 0 on success.
 	 */
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 77b05b1edc..4ea4204fd7 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -9906,6 +9906,7 @@ mdb_env_copyfd1(MDB_env *env, HANDLE fd)
 	mdb_copy my = {0};
 	MDB_txn *txn = NULL;
 	pthread_t thr;
+	pgno_t root, new_root;
 	int rc = MDB_SUCCESS;
 
 #ifdef _WIN32
@@ -9963,10 +9964,12 @@ mdb_env_copyfd1(MDB_env *env, HANDLE fd)
 	*(MDB_meta *)METADATA(mp) = *mm;
 	mm = (MDB_meta *)METADATA(mp);
 
-	/* Count the number of free pages, subtract from lastpg to find
-	 * number of active pages
-	 */
-	{
+	/* Set metapage 1 with current main DB */
+	root = new_root = txn->mt_dbs[MAIN_DBI].md_root;
+	if (root != P_INVALID) {
+		/* Count free pages + freeDB pages.  Subtract from last_pg
+		 * to find the new last_pg, which also becomes the new root.
+		 */
 		MDB_ID freecount = 0;
 		MDB_cursor mc;
 		MDB_val key, data;
@@ -9979,19 +9982,26 @@ mdb_env_copyfd1(MDB_env *env, HANDLE fd)
 			txn->mt_dbs[FREE_DBI].md_leaf_pages +
 			txn->mt_dbs[FREE_DBI].md_overflow_pages;
 
-		/* Set metapage 1 */
-		mm->mm_last_pg = txn->mt_next_pgno - freecount - 1;
+		new_root = txn->mt_next_pgno - 1 - freecount;
+		mm->mm_last_pg = new_root;
 		mm->mm_dbs[MAIN_DBI] = txn->mt_dbs[MAIN_DBI];
-		if (mm->mm_last_pg > NUM_METAS-1) {
-			mm->mm_dbs[MAIN_DBI].md_root = mm->mm_last_pg;
-			mm->mm_txnid = 1;
-		} else {
-			mm->mm_dbs[MAIN_DBI].md_root = P_INVALID;
-		}
+		mm->mm_dbs[MAIN_DBI].md_root = new_root;
+	} else {
+		/* When the DB is empty, handle it specially to
+		 * fix any breakage like page leaks from ITS#8174.
+		 */
+		mm->mm_dbs[MAIN_DBI].md_flags = txn->mt_dbs[MAIN_DBI].md_flags;
 	}
+	if (root != P_INVALID || mm->mm_dbs[MAIN_DBI].md_flags) {
+		mm->mm_txnid = 1;		/* use metapage 1 */
+	}
+
 	my.mc_wlen[0] = env->me_psize * NUM_METAS;
 	my.mc_txn = txn;
-	rc = mdb_env_cwalk(&my, &txn->mt_dbs[MAIN_DBI].md_root, 0);
+	rc = mdb_env_cwalk(&my, &root, 0);
+	if (rc == MDB_SUCCESS && root != new_root) {
+		rc = MDB_INCOMPATIBLE;	/* page leak or corrupt DB */
+	}
 
 finish:
 	if (rc)
diff --git a/libraries/liblmdb/mdb_copy.1 b/libraries/liblmdb/mdb_copy.1
index 258affbfb4..4387ac3a14 100644
--- a/libraries/liblmdb/mdb_copy.1
+++ b/libraries/liblmdb/mdb_copy.1
@@ -36,6 +36,7 @@ Write the library version number to the standard output, and exit.
 Compact while copying. Only current data pages will be copied; freed
 or unused pages will be omitted from the copy. This option will
 slow down the backup process as it is more CPU-intensive.
+Currently it fails if the environment has suffered a page leak.
 .TP
 .BR \-n
 Open LDMB environment(s) which do not use subdirectories.

From 291c69ddbd593ce86c888718745f6d78f6bd7222 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Wed, 29 Jun 2016 06:25:37 +0200
Subject: [PATCH 239/504] ITS#8209 Tweak previous fixes

Some _aligned_malloc() doc seems to think arg NULL = user error.
Don't know if posix_memalign() pointer is defined after failure.
---
 libraries/liblmdb/mdb.c | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 4ea4204fd7..3777ab314f 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -9922,9 +9922,10 @@ mdb_env_copyfd1(MDB_env *env, HANDLE fd)
 		goto done;
 	}
 #else
-	if ((rc = pthread_mutex_init(&my.mc_mutex, NULL)) ||
-		(rc = pthread_cond_init(&my.mc_cond, NULL)))
+	if ((rc = pthread_mutex_init(&my.mc_mutex, NULL)) != 0)
 		return rc;
+	if ((rc = pthread_cond_init(&my.mc_cond, NULL)) != 0)
+		goto done2;
 #ifdef HAVE_MEMALIGN
 	my.mc_wbuf[0] = memalign(env->me_os_psize, MDB_WBUF*2);
 	if (my.mc_wbuf[0] == NULL) {
@@ -9932,9 +9933,12 @@ mdb_env_copyfd1(MDB_env *env, HANDLE fd)
 		goto done;
 	}
 #else
-	rc = posix_memalign((void **)&my.mc_wbuf[0], env->me_os_psize, MDB_WBUF*2);
-	if (rc)
-		goto done;
+	{
+		void *p;
+		if ((rc = posix_memalign(&p, env->me_os_psize, MDB_WBUF*2)) != 0)
+			goto done;
+		my.mc_wbuf[0] = p;
+	}
 #endif
 #endif
 	memset(my.mc_wbuf[0], 0, MDB_WBUF*2);
@@ -10012,13 +10016,14 @@ finish:
 
 done:
 #ifdef _WIN32
+	if (my.mc_wbuf[0]) _aligned_free(my.mc_wbuf[0]);
 	if (my.mc_cond)  CloseHandle(my.mc_cond);
 	if (my.mc_mutex) CloseHandle(my.mc_mutex);
-	_aligned_free(my.mc_wbuf[0]);
 #else
-	pthread_cond_destroy(&my.mc_cond);
-	pthread_mutex_destroy(&my.mc_mutex);
 	free(my.mc_wbuf[0]);
+	pthread_cond_destroy(&my.mc_cond);
+done2:
+	pthread_mutex_destroy(&my.mc_mutex);
 #endif
 	return rc ? rc : my.mc_error;
 }

From 84610e65da85d483f9461b4bdc300a082f004de2 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Thu, 14 Jul 2016 05:53:21 +0200
Subject: [PATCH 240/504] Add error MDB_PROBLEM, replace some MDB_CORRUPTED

When problem is most likely in txn, not on disk.
---
 libraries/liblmdb/lmdb.h | 4 +++-
 libraries/liblmdb/mdb.c  | 9 +++++----
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 85dc50d580..d30ab648b3 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -456,8 +456,10 @@ typedef enum MDB_cursor_op {
 #define MDB_BAD_VALSIZE		(-30781)
 	/** The specified DBI was changed unexpectedly */
 #define MDB_BAD_DBI		(-30780)
+	/** Unexpected problem - txn should abort */
+#define MDB_PROBLEM		(-30779)
 	/** The last defined error code */
-#define MDB_LAST_ERRCODE	MDB_BAD_DBI
+#define MDB_LAST_ERRCODE	MDB_PROBLEM
 /** @} */
 
 /** @brief Statistics for a database in the environment */
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 3777ab314f..df31780d0c 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1565,6 +1565,7 @@ static char *const mdb_errstr[] = {
 	"MDB_BAD_TXN: Transaction must abort, has a child, or is invalid",
 	"MDB_BAD_VALSIZE: Unsupported size of key/DB name/data, or wrong DUPFIXED size",
 	"MDB_BAD_DBI: The specified DBI handle was closed/changed unexpectedly",
+	"MDB_PROBLEM: Unexpected problem - txn should abort",
 };
 
 char *
@@ -2001,7 +2002,7 @@ mdb_page_loose(MDB_cursor *mc, MDB_page *mp)
 					if (mp != dl[x].mptr) { /* bad cursor? */
 						mc->mc_flags &= ~(C_INITIALIZED|C_EOF);
 						txn->mt_flags |= MDB_TXN_ERROR;
-						return MDB_CORRUPTED;
+						return MDB_PROBLEM;
 					}
 					/* ok, it's ours */
 					loose = 1;
@@ -2611,7 +2612,7 @@ mdb_page_touch(MDB_cursor *mc)
 				if (mp != dl[x].mptr) { /* bad cursor? */
 					mc->mc_flags &= ~(C_INITIALIZED|C_EOF);
 					txn->mt_flags |= MDB_TXN_ERROR;
-					return MDB_CORRUPTED;
+					return MDB_PROBLEM;
 				}
 				return 0;
 			}
@@ -6263,7 +6264,7 @@ mdb_ovpage_free(MDB_cursor *mc, MDB_page *mp)
 				j = ++(dl[0].mid);
 				dl[j] = ix;		/* Unsorted. OK when MDB_TXN_ERROR. */
 				txn->mt_flags |= MDB_TXN_ERROR;
-				return MDB_CORRUPTED;
+				return MDB_PROBLEM;
 			}
 		}
 		txn->mt_dirty_room++;
@@ -7656,7 +7657,7 @@ put_sub:
 		return rc;
 bad_sub:
 		if (rc == MDB_KEYEXIST)	/* should not happen, we deleted that item */
-			rc = MDB_CORRUPTED;
+			rc = MDB_PROBLEM;
 	}
 	mc->mc_txn->mt_flags |= MDB_TXN_ERROR;
 	return rc;

From 32764bcb52e70588982d576cb16f435705c11279 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 23 Jul 2016 12:08:12 +0200
Subject: [PATCH 241/504] Factor out MDB_SIZE_MAX, MDB_FMT_Y, MDB_FMT_Z

---
 libraries/liblmdb/lmdb.h     | 19 +++++++++++++++++++
 libraries/liblmdb/mdb.c      | 13 ++-----------
 libraries/liblmdb/mdb_dump.c | 15 +--------------
 libraries/liblmdb/mdb_load.c | 16 ++--------------
 libraries/liblmdb/mdb_stat.c | 16 ++--------------
 libraries/liblmdb/midl.h     |  9 ++-------
 6 files changed, 28 insertions(+), 60 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index d30ab648b3..92dd753d36 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -179,13 +179,32 @@ typedef	int	mdb_mode_t;
 typedef	mode_t	mdb_mode_t;
 #endif
 
+#ifdef _WIN32
+# define MDB_FMT_Z	"I"
+#else
+# define MDB_FMT_Z	"z"			/**< printf/scanf format modifier for size_t */
+#endif
+
 #ifdef MDB_VL32
 typedef uint64_t	mdb_size_t;
+#define MDB_SIZE_MAX UINT64_MAX
+#ifdef _WIN32
+# define MDB_FMT_Y	"I64"
+#else
+# define MDB_FMT_Y	"ll"
+#endif
 #define mdb_env_create	mdb_env_create_vl32	/**< Prevent mixing with non-VL32 builds */
 #else
 typedef size_t	mdb_size_t;
+# define MDB_SIZE_MAX	SIZE_MAX	/**< max #mdb_size_t */
+# define MDB_FMT_Y		MDB_FMT_Z	/**< Obsolescent, see #MDB_PRIz()/#MDB_SCNz() */
 #endif
 
+/** #mdb_size_t printf formats, \b t = one of [diouxX] without quotes */
+#define MDB_PRIz(t)	MDB_FMT_Y #t
+/** #mdb_size_t scanf formats, \b t = one of [dioux] without quotes */
+#define MDB_SCNz(t)	MDB_FMT_Y #t
+
 /** An abstraction for a file handle.
  *	On POSIX systems file handles are small integers. On Windows
  *	they're opaque pointers.
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index df31780d0c..01c259f849 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -355,12 +355,10 @@ typedef HANDLE mdb_mutex_t, mdb_mutexref_t;
 #else
 #define MDB_PROCESS_QUERY_LIMITED_INFORMATION 0x1000
 #endif
-#define	Z	"I"
 #else
 #define THREAD_RET	void *
 #define THREAD_CREATE(thr,start,arg)	pthread_create(&thr,NULL,start,arg)
 #define THREAD_FINISH(thr)	pthread_join(thr,NULL)
-#define	Z	"z"			/**< printf format modifier for size_t */
 
 	/** For MDB_LOCK_FORMAT: True if readers take a pid lock in the lockfile */
 #define MDB_PIDLOCK			1
@@ -458,15 +456,8 @@ typedef pthread_mutex_t mdb_mutex_t[1], *mdb_mutexref_t;
 #define	GET_PAGESIZE(x)	((x) = sysconf(_SC_PAGE_SIZE))
 #endif
 
-#ifdef MDB_VL32
-#ifdef _WIN32
-#define	Y	"I64"
-#else
-#define	Y	"ll"
-#endif
-#else
-#define Y	Z
-#endif
+#define	Z	MDB_FMT_Z	/**< printf/scanf format modifier for size_t */
+#define	Y	MDB_FMT_Y	/**< printf/scanf format modifier for #mdb_size_t */
 
 #if defined(_WIN32) || defined(MDB_USE_POSIX_SEM)
 #define MNAME_LEN	32
diff --git a/libraries/liblmdb/mdb_dump.c b/libraries/liblmdb/mdb_dump.c
index 5f3125fa2a..538b243950 100644
--- a/libraries/liblmdb/mdb_dump.c
+++ b/libraries/liblmdb/mdb_dump.c
@@ -20,20 +20,7 @@
 #include <signal.h>
 #include "lmdb.h"
 
-#ifdef _WIN32
-#define Z	"I"
-#else
-#define Z	"z"
-#endif
-#ifdef MDB_VL32
-#ifdef _WIN32
-#define	Y	"I64"
-#else
-#define	Y	"ll"
-#endif
-#else
-#define Y	Z
-#endif
+#define Y	MDB_FMT_Y
 
 #define PRINT	1
 static int mode;
diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c
index 27aee40290..fe1d846b53 100644
--- a/libraries/liblmdb/mdb_load.c
+++ b/libraries/liblmdb/mdb_load.c
@@ -38,20 +38,8 @@ static MDB_envinfo info;
 
 static MDB_val kbuf, dbuf;
 
-#ifdef _WIN32
-#define Z	"I"
-#else
-#define Z	"z"
-#endif
-#ifdef MDB_VL32
-#ifdef _WIN32
-#define	Y	"I64"
-#else
-#define	Y	"ll"
-#endif
-#else
-#define Y	Z
-#endif
+#define Z	MDB_FMT_Z
+#define Y	MDB_FMT_Y
 
 #define STRLENOF(s)	(sizeof(s)-1)
 
diff --git a/libraries/liblmdb/mdb_stat.c b/libraries/liblmdb/mdb_stat.c
index de5c3b514c..11c9e481d7 100644
--- a/libraries/liblmdb/mdb_stat.c
+++ b/libraries/liblmdb/mdb_stat.c
@@ -17,20 +17,8 @@
 #include <unistd.h>
 #include "lmdb.h"
 
-#ifdef	_WIN32
-#define	Z	"I"
-#else
-#define	Z	"z"
-#endif
-#ifdef MDB_VL32
-#ifdef _WIN32
-#define	Y	"I64"
-#else
-#define	Y	"ll"
-#endif
-#else
-#define Y	Z
-#endif
+#define Z	MDB_FMT_Z
+#define Y	MDB_FMT_Y
 
 static void prstat(MDB_stat *ms)
 {
diff --git a/libraries/liblmdb/midl.h b/libraries/liblmdb/midl.h
index b0d518ef03..dc532c44ff 100644
--- a/libraries/liblmdb/midl.h
+++ b/libraries/liblmdb/midl.h
@@ -26,8 +26,7 @@
 #ifndef _MDB_MIDL_H_
 #define _MDB_MIDL_H_
 
-#include <stddef.h>
-#include <inttypes.h>
+#include "lmdb.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -43,11 +42,7 @@ extern "C" {
 	/** A generic unsigned ID number. These were entryIDs in back-bdb.
 	 *	Preferably it should have the same size as a pointer.
 	 */
-#ifdef MDB_VL32
-typedef uint64_t MDB_ID;
-#else
-typedef size_t MDB_ID;
-#endif
+typedef mdb_size_t MDB_ID;
 
 	/** An IDL is an ID List, a sorted array of IDs. The first
 	 * element of the array is a counter for how many actual

From dff8bafb36eef019b1b121eb10fbd259197fa01f Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 23 Jul 2016 12:11:34 +0200
Subject: [PATCH 242/504] Factor some MDB_VL32-related '#if's out to macros

Add MC_OVPG() + MC_SET_OVPG(), NEED_CMP_CLONG(), MDB_CURSOR_UNREF().
---
 libraries/liblmdb/mdb.c | 103 +++++++++++++++-------------------------
 1 file changed, 38 insertions(+), 65 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 01c259f849..258dd39ecf 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1304,6 +1304,11 @@ struct MDB_cursor {
 	indx_t		mc_ki[CURSOR_STACK];	/**< stack of page indices */
 #ifdef MDB_VL32
 	MDB_page	*mc_ovpg;		/**< a referenced overflow page */
+#	define MC_OVPG(mc)			((mc)->mc_ovpg)
+#	define MC_SET_OVPG(mc, pg)	((mc)->mc_ovpg = (pg))
+#else
+#	define MC_OVPG(mc)			((MDB_page *)0)
+#	define MC_SET_OVPG(mc, pg)	((void)0)
 #endif
 };
 
@@ -1516,6 +1521,11 @@ static MDB_cmp_func	mdb_cmp_memn, mdb_cmp_memnr, mdb_cmp_int, mdb_cmp_cint, mdb_
 # define mdb_cmp_clong mdb_cmp_cint
 #endif
 
+/** True if we need #mdb_cmp_clong() instead of \b cmp for #MDB_INTEGERDUP */
+#define NEED_CMP_CLONG(cmp, ksize) \
+	(UINT_MAX < MDB_SIZE_MAX && \
+	 (cmp) == mdb_cmp_int && (ksize) == sizeof(mdb_size_t))
+
 #ifdef _WIN32
 static SECURITY_DESCRIPTOR mdb_null_sd;
 static SECURITY_ATTRIBUTES mdb_all_sa;
@@ -1842,10 +1852,8 @@ int
 mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b)
 {
 	MDB_cmp_func *dcmp = txn->mt_dbxs[dbi].md_dcmp;
-#if UINT_MAX < SIZE_MAX || defined(MDB_VL32)
-	if (dcmp == mdb_cmp_int && a->mv_size == sizeof(mdb_size_t))
+	if (NEED_CMP_CLONG(dcmp, a->mv_size))
 		dcmp = mdb_cmp_clong;
-#endif
 	return dcmp(a, b);
 }
 
@@ -1960,8 +1968,14 @@ mdb_cursor_unref(MDB_cursor *mc)
 	mc->mc_pg[0] = NULL;
 	mc->mc_flags &= ~C_INITIALIZED;
 }
+#define MDB_CURSOR_UNREF(mc, force) \
+	(((force) || ((mc)->mc_flags & C_INITIALIZED)) \
+	 ? mdb_cursor_unref(mc) \
+	 : (void)0)
+
 #else
 #define MDB_PAGE_UNREF(txn, mp)
+#define MDB_CURSOR_UNREF(mc, force) ((void)0)
 #endif /* MDB_VL32 */
 
 /** Loosen or free a single page.
@@ -6292,12 +6306,10 @@ mdb_node_read(MDB_cursor *mc, MDB_node *leaf, MDB_val *data)
 	pgno_t		 pgno;
 	int rc;
 
-#ifdef MDB_VL32
-	if (mc->mc_ovpg) {
-		MDB_PAGE_UNREF(mc->mc_txn, mc->mc_ovpg);
-		mc->mc_ovpg = 0;
+	if (MC_OVPG(mc)) {
+		MDB_PAGE_UNREF(mc->mc_txn, MC_OVPG(mc));
+		MC_SET_OVPG(mc, NULL);
 	}
-#endif
 	if (!F_ISSET(leaf->mn_flags, F_BIGDATA)) {
 		data->mv_size = NODEDSZ(leaf);
 		data->mv_data = NODEDATA(leaf);
@@ -6313,9 +6325,7 @@ mdb_node_read(MDB_cursor *mc, MDB_node *leaf, MDB_val *data)
 		return rc;
 	}
 	data->mv_data = METADATA(omp);
-#ifdef MDB_VL32
-	mc->mc_ovpg = omp;
-#endif
+	MC_SET_OVPG(mc, omp);
 
 	return MDB_SUCCESS;
 }
@@ -6339,14 +6349,10 @@ mdb_get(MDB_txn *txn, MDB_dbi dbi,
 
 	mdb_cursor_init(&mc, txn, dbi, &mx);
 	rc = mdb_cursor_set(&mc, key, data, MDB_SET, &exact);
-#ifdef MDB_VL32
-	{
-		/* unref all the pages - caller must copy the data
-		 * before doing anything else
-		 */
-		mdb_cursor_unref(&mc);
-	}
-#endif
+	/* unref all the pages when MDB_VL32 - caller must copy the data
+	 * before doing anything else
+	 */
+	MDB_CURSOR_UNREF(&mc, 1);
 	return rc;
 }
 
@@ -6443,13 +6449,9 @@ mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
 					return rc;
 				}
 			}
-#ifdef MDB_VL32
 			else {
-				if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) {
-					mdb_cursor_unref(&mc->mc_xcursor->mx_cursor);
-				}
+				MDB_CURSOR_UNREF(&mc->mc_xcursor->mx_cursor, 0);
 			}
-#endif
 		} else {
 			mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF);
 			if (op == MDB_NEXT_DUP)
@@ -6536,13 +6538,9 @@ mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
 					return rc;
 				}
 			}
-#ifdef MDB_VL32
 			else {
-				if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) {
-					mdb_cursor_unref(&mc->mc_xcursor->mx_cursor);
-				}
+				MDB_CURSOR_UNREF(&mc->mc_xcursor->mx_cursor, 0);
 			}
-#endif
 		} else {
 			mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF);
 			if (op == MDB_PREV_DUP)
@@ -6744,11 +6742,8 @@ set1:
 		return MDB_SUCCESS;
 	}
 
-#ifdef MDB_VL32
-	if (mc->mc_xcursor && mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) {
-		mdb_cursor_unref(&mc->mc_xcursor->mx_cursor);
-	}
-#endif
+	if (mc->mc_xcursor)
+		MDB_CURSOR_UNREF(&mc->mc_xcursor->mx_cursor, 0);
 	if (F_ISSET(leaf->mn_flags, F_DUPDATA)) {
 		mdb_xcursor_init1(mc, leaf);
 	}
@@ -6774,10 +6769,8 @@ set1:
 			if ((rc = mdb_node_read(mc, leaf, &olddata)) != MDB_SUCCESS)
 				return rc;
 			dcmp = mc->mc_dbx->md_dcmp;
-#if UINT_MAX < SIZE_MAX || defined(MDB_VL32)
-			if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(mdb_size_t))
+			if (NEED_CMP_CLONG(dcmp, olddata.mv_size))
 				dcmp = mdb_cmp_clong;
-#endif
 			rc = dcmp(data, &olddata);
 			if (rc) {
 				if (op == MDB_GET_BOTH || rc > 0)
@@ -6810,11 +6803,7 @@ mdb_cursor_first(MDB_cursor *mc, MDB_val *key, MDB_val *data)
 	MDB_node	*leaf;
 
 	if (mc->mc_xcursor) {
-#ifdef MDB_VL32
-		if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) {
-			mdb_cursor_unref(&mc->mc_xcursor->mx_cursor);
-		}
-#endif
+		MDB_CURSOR_UNREF(&mc->mc_xcursor->mx_cursor, 0);
 		mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF);
 	}
 
@@ -6860,11 +6849,7 @@ mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data)
 	MDB_node	*leaf;
 
 	if (mc->mc_xcursor) {
-#ifdef MDB_VL32
-		if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) {
-			mdb_cursor_unref(&mc->mc_xcursor->mx_cursor);
-		}
-#endif
+		MDB_CURSOR_UNREF(&mc->mc_xcursor->mx_cursor, 0);
 		mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF);
 	}
 
@@ -7320,10 +7305,8 @@ more:
 				if (flags == MDB_CURRENT)
 					goto current;
 				dcmp = mc->mc_dbx->md_dcmp;
-#if UINT_MAX < SIZE_MAX || defined(MDB_VL32)
-				if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(mdb_size_t))
+				if (NEED_CMP_CLONG(dcmp, olddata.mv_size))
 					dcmp = mdb_cmp_clong;
-#endif
 				/* does data match? */
 				if (!dcmp(data, &olddata)) {
 					if (flags & (MDB_NODUPDATA|MDB_APPENDDUP))
@@ -8107,9 +8090,7 @@ mdb_xcursor_init0(MDB_cursor *mc)
 	mx->mx_cursor.mc_dbflag = &mx->mx_dbflag;
 	mx->mx_cursor.mc_snum = 0;
 	mx->mx_cursor.mc_top = 0;
-#ifdef MDB_VL32
-	mx->mx_cursor.mc_ovpg = 0;
-#endif
+	MC_SET_OVPG(&mx->mx_cursor, NULL);
 	mx->mx_cursor.mc_flags = C_SUB | (mc->mc_flags & (C_ORIG_RDONLY|C_WRITEMAP));
 	mx->mx_dbx.md_name.mv_size = 0;
 	mx->mx_dbx.md_name.mv_data = NULL;
@@ -8160,10 +8141,8 @@ mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node)
 	DPRINTF(("Sub-db -%u root page %"Y"u", mx->mx_cursor.mc_dbi,
 		mx->mx_db.md_root));
 	mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DIRTY; /* DB_DIRTY guides mdb_cursor_touch */
-#if UINT_MAX < SIZE_MAX || defined(MDB_VL32)
-	if (mx->mx_dbx.md_cmp == mdb_cmp_int && mx->mx_db.md_pad == sizeof(mdb_size_t))
+	if (NEED_CMP_CLONG(mx->mx_dbx.md_cmp, mx->mx_db.md_pad))
 		mx->mx_dbx.md_cmp = mdb_cmp_clong;
-#endif
 }
 
 
@@ -8213,9 +8192,7 @@ mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx)
 	mc->mc_top = 0;
 	mc->mc_pg[0] = 0;
 	mc->mc_ki[0] = 0;
-#ifdef MDB_VL32
-	mc->mc_ovpg = 0;
-#endif
+	MC_SET_OVPG(mc, NULL);
 	mc->mc_flags = txn->mt_flags & (C_ORIG_RDONLY|C_WRITEMAP);
 	if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) {
 		mdb_tassert(txn, mx != NULL);
@@ -8830,9 +8807,7 @@ mdb_cursor_copy(const MDB_cursor *csrc, MDB_cursor *cdst)
 	cdst->mc_snum = csrc->mc_snum;
 	cdst->mc_top = csrc->mc_top;
 	cdst->mc_flags = csrc->mc_flags;
-#ifdef MDB_VL32
-	cdst->mc_ovpg = csrc->mc_ovpg;
-#endif
+	MC_SET_OVPG(cdst, MC_OVPG(csrc));
 
 	for (i=0; i<csrc->mc_snum; i++) {
 		cdst->mc_pg[i] = csrc->mc_pg[i];
@@ -10605,10 +10580,8 @@ pop:
 done:
 		if (rc)
 			txn->mt_flags |= MDB_TXN_ERROR;
-#ifdef MDB_VL32
 		/* drop refcount for mx's pages */
-		mdb_cursor_unref(&mx);
-#endif
+		MDB_CURSOR_UNREF(&mx, 0);
 	} else if (rc == MDB_NOTFOUND) {
 		rc = MDB_SUCCESS;
 	}

From 12ad38d67f722d22a8ab3793dc2967d1247d3fea Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 23 Jul 2016 12:16:20 +0200
Subject: [PATCH 243/504] Fix size_t/formats -> mdb_size_t for MDB_VL32

---
 libraries/liblmdb/mdb.c      |  8 ++++----
 libraries/liblmdb/mdb_stat.c | 12 ++++++------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 258dd39ecf..ba5a39dd8b 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -228,7 +228,7 @@ union semun {
 
 #if (BYTE_ORDER == LITTLE_ENDIAN) == (BYTE_ORDER == BIG_ENDIAN)
 # error "Unknown or unsupported endianness (BYTE_ORDER)"
-#elif (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF
+#elif (-6 & 5) || CHAR_BIT!=8 || UINT_MAX!=0xffffffff || MDB_SIZE_MAX%UINT_MAX
 # error "Two's complement, reasonably sized integer types, please"
 #endif
 
@@ -1028,7 +1028,7 @@ typedef struct MDB_node {
 #ifdef MISALIGNED_OK
 #define COPY_PGNO(dst,src)	dst = src
 #else
-#if SIZE_MAX > 4294967295UL
+#if MDB_SIZE_MAX > 0xffffffffU
 #define COPY_PGNO(dst,src)	do { \
 	unsigned short *s, *d;	\
 	s = (unsigned short *)&(src);	\
@@ -3551,7 +3551,7 @@ mdb_page_flush(MDB_txn *txn, int keep)
 		 * the write offset, to at least save the overhead of a Seek
 		 * system call.
 		 */
-		DPRINTF(("committing page %"Z"u", pgno));
+		DPRINTF(("committing page %"Y"u", pgno));
 		memset(&ov, 0, sizeof(ov));
 		ov.Offset = pos & 0xffffffff;
 		ov.OffsetHigh = pos >> 16 >> 16;
@@ -8165,7 +8165,7 @@ mdb_xcursor_init2(MDB_cursor *mc, MDB_xcursor *src_mx, int new_dupdata)
 		mx->mx_cursor.mc_flags |= C_INITIALIZED;
 		mx->mx_cursor.mc_ki[0] = 0;
 		mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DIRTY; /* DB_DIRTY guides mdb_cursor_touch */
-#if UINT_MAX < SIZE_MAX
+#if UINT_MAX < MDB_SIZE_MAX	/* matches mdb_xcursor_init1:NEED_CMP_CLONG() */
 		mx->mx_dbx.md_cmp = src_mx->mx_dbx.md_cmp;
 #endif
 	} else if (!(mx->mx_cursor.mc_flags & C_INITIALIZED)) {
diff --git a/libraries/liblmdb/mdb_stat.c b/libraries/liblmdb/mdb_stat.c
index 11c9e481d7..5c57072db4 100644
--- a/libraries/liblmdb/mdb_stat.c
+++ b/libraries/liblmdb/mdb_stat.c
@@ -153,7 +153,7 @@ int main(int argc, char *argv[])
 	if (freinfo) {
 		MDB_cursor *cursor;
 		MDB_val key, data;
-		size_t pages = 0, *iptr;
+		mdb_size_t pages = 0, *iptr;
 
 		printf("Freelist Status\n");
 		dbi = 0;
@@ -173,7 +173,7 @@ int main(int argc, char *argv[])
 			pages += *iptr;
 			if (freinfo > 1) {
 				char *bad = "";
-				size_t pg, prev;
+				mdb_size_t pg, prev;
 				ssize_t i, j, span = 0;
 				j = *iptr++;
 				for (i = j, prev = 1; --i >= 0; ) {
@@ -184,20 +184,20 @@ int main(int argc, char *argv[])
 					pg += span;
 					for (; i >= span && iptr[i-span] == pg; span++, pg++) ;
 				}
-				printf("    Transaction %"Z"u, %"Z"d pages, maxspan %"Z"d%s\n",
-					*(size_t *)key.mv_data, j, span, bad);
+				printf("    Transaction %"Y"u, %"Z"d pages, maxspan %"Z"d%s\n",
+					*(mdb_size_t *)key.mv_data, j, span, bad);
 				if (freinfo > 2) {
 					for (--j; j >= 0; ) {
 						pg = iptr[j];
 						for (span=1; --j >= 0 && iptr[j] == pg+span; span++) ;
-						printf(span>1 ? "     %9"Z"u[%"Z"d]\n" : "     %9"Z"u\n",
+						printf(span>1 ? "     %9"Y"u[%"Z"d]\n" : "     %9"Y"u\n",
 							pg, span);
 					}
 				}
 			}
 		}
 		mdb_cursor_close(cursor);
-		printf("  Free pages: %"Z"u\n", pages);
+		printf("  Free pages: %"Y"u\n", pages);
 	}
 
 	rc = mdb_open(txn, subname, 0, &dbi);

From 0842f998ee49b7891483c601fc4f8ad7d83b30d3 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 23 Jul 2016 12:17:04 +0200
Subject: [PATCH 244/504] Use mdb_size_t for line numbers in mdb_load

This matches the mdb_size_t entry counts.
---
 libraries/liblmdb/mdb_load.c | 29 ++++++++++++++---------------
 1 file changed, 14 insertions(+), 15 deletions(-)

diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c
index fe1d846b53..3b55a946d1 100644
--- a/libraries/liblmdb/mdb_load.c
+++ b/libraries/liblmdb/mdb_load.c
@@ -25,7 +25,7 @@ static int mode;
 
 static char *subname = NULL;
 
-static size_t lineno;
+static mdb_size_t lineno;
 static int version;
 
 static int flags;
@@ -38,7 +38,6 @@ static MDB_envinfo info;
 
 static MDB_val kbuf, dbuf;
 
-#define Z	MDB_FMT_Z
 #define Y	MDB_FMT_Y
 
 #define STRLENOF(s)	(sizeof(s)-1)
@@ -70,7 +69,7 @@ static void readhdr(void)
 		if (!strncmp(dbuf.mv_data, "VERSION=", STRLENOF("VERSION="))) {
 			version=atoi((char *)dbuf.mv_data+STRLENOF("VERSION="));
 			if (version > 3) {
-				fprintf(stderr, "%s: line %" Z "d: unsupported VERSION %d\n",
+				fprintf(stderr, "%s: line %" Y "u: unsupported VERSION %d\n",
 					prog, lineno, version);
 				exit(EXIT_FAILURE);
 			}
@@ -80,7 +79,7 @@ static void readhdr(void)
 			if (!strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "print", STRLENOF("print")))
 				mode |= PRINT;
 			else if (strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "bytevalue", STRLENOF("bytevalue"))) {
-				fprintf(stderr, "%s: line %" Z "d: unsupported FORMAT %s\n",
+				fprintf(stderr, "%s: line %" Y "u: unsupported FORMAT %s\n",
 					prog, lineno, (char *)dbuf.mv_data+STRLENOF("FORMAT="));
 				exit(EXIT_FAILURE);
 			}
@@ -91,7 +90,7 @@ static void readhdr(void)
 			subname = strdup((char *)dbuf.mv_data+STRLENOF("database="));
 		} else if (!strncmp(dbuf.mv_data, "type=", STRLENOF("type="))) {
 			if (strncmp((char *)dbuf.mv_data+STRLENOF("type="), "btree", STRLENOF("btree")))  {
-				fprintf(stderr, "%s: line %" Z "d: unsupported type %s\n",
+				fprintf(stderr, "%s: line %" Y "u: unsupported type %s\n",
 					prog, lineno, (char *)dbuf.mv_data+STRLENOF("type="));
 				exit(EXIT_FAILURE);
 			}
@@ -101,7 +100,7 @@ static void readhdr(void)
 			if (ptr) *ptr = '\0';
 			i = sscanf((char *)dbuf.mv_data+STRLENOF("mapaddr="), "%p", &info.me_mapaddr);
 			if (i != 1) {
-				fprintf(stderr, "%s: line %" Z "d: invalid mapaddr %s\n",
+				fprintf(stderr, "%s: line %" Y "u: invalid mapaddr %s\n",
 					prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapaddr="));
 				exit(EXIT_FAILURE);
 			}
@@ -111,7 +110,7 @@ static void readhdr(void)
 			if (ptr) *ptr = '\0';
 			i = sscanf((char *)dbuf.mv_data+STRLENOF("mapsize="), "%" Y "u", &info.me_mapsize);
 			if (i != 1) {
-				fprintf(stderr, "%s: line %" Z "d: invalid mapsize %s\n",
+				fprintf(stderr, "%s: line %" Y "u: invalid mapsize %s\n",
 					prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapsize="));
 				exit(EXIT_FAILURE);
 			}
@@ -121,7 +120,7 @@ static void readhdr(void)
 			if (ptr) *ptr = '\0';
 			i = sscanf((char *)dbuf.mv_data+STRLENOF("maxreaders="), "%u", &info.me_maxreaders);
 			if (i != 1) {
-				fprintf(stderr, "%s: line %" Z "d: invalid maxreaders %s\n",
+				fprintf(stderr, "%s: line %" Y "u: invalid maxreaders %s\n",
 					prog, lineno, (char *)dbuf.mv_data+STRLENOF("maxreaders="));
 				exit(EXIT_FAILURE);
 			}
@@ -137,12 +136,12 @@ static void readhdr(void)
 			if (!dbflags[i].bit) {
 				ptr = memchr(dbuf.mv_data, '=', dbuf.mv_size);
 				if (!ptr) {
-					fprintf(stderr, "%s: line %" Z "d: unexpected format\n",
+					fprintf(stderr, "%s: line %" Y "u: unexpected format\n",
 						prog, lineno);
 					exit(EXIT_FAILURE);
 				} else {
 					*ptr = '\0';
-					fprintf(stderr, "%s: line %" Z "d: unrecognized keyword ignored: %s\n",
+					fprintf(stderr, "%s: line %" Y "u: unrecognized keyword ignored: %s\n",
 						prog, lineno, (char *)dbuf.mv_data);
 				}
 			}
@@ -152,7 +151,7 @@ static void readhdr(void)
 
 static void badend(void)
 {
-	fprintf(stderr, "%s: line %" Z "d: unexpected end of input\n",
+	fprintf(stderr, "%s: line %" Y "u: unexpected end of input\n",
 		prog, lineno);
 }
 
@@ -210,7 +209,7 @@ badend:
 		buf->mv_data = realloc(buf->mv_data, buf->mv_size*2);
 		if (!buf->mv_data) {
 			Eof = 1;
-			fprintf(stderr, "%s: line %" Z "d: out of memory, line too long\n",
+			fprintf(stderr, "%s: line %" Y "u: out of memory, line too long\n",
 				prog, lineno);
 			return EOF;
 		}
@@ -402,7 +401,7 @@ int main(int argc, char *argv[])
 
 			rc = readline(&data, &dbuf);
 			if (rc) {
-				fprintf(stderr, "%s: line %" Z "d: failed to read key value\n", prog, lineno);
+				fprintf(stderr, "%s: line %" Y "u: failed to read key value\n", prog, lineno);
 				goto txn_abort;
 			}
 
@@ -417,7 +416,7 @@ int main(int argc, char *argv[])
 			if (batch == 100) {
 				rc = mdb_txn_commit(txn);
 				if (rc) {
-					fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n",
+					fprintf(stderr, "%s: line %" Y "u: txn_commit: %s\n",
 						prog, lineno, mdb_strerror(rc));
 					goto env_close;
 				}
@@ -437,7 +436,7 @@ int main(int argc, char *argv[])
 		rc = mdb_txn_commit(txn);
 		txn = NULL;
 		if (rc) {
-			fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n",
+			fprintf(stderr, "%s: line %" Y "u: txn_commit: %s\n",
 				prog, lineno, mdb_strerror(rc));
 			goto env_close;
 		}

From f25d716513944b667d430beece3d616d326e0cf2 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 23 Jul 2016 12:18:01 +0200
Subject: [PATCH 245/504] Fix MDB_INTEGERKEY doc of integer types

---
 libraries/liblmdb/lmdb.h | 6 ++++--
 libraries/liblmdb/mdb.c  | 2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 92dd753d36..c21115d808 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -339,7 +339,8 @@ typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *rel
 #define MDB_REVERSEKEY	0x02
 	/** use sorted duplicates */
 #define MDB_DUPSORT		0x04
-	/** numeric keys in native byte order: either unsigned int or size_t.
+	/** numeric keys in native byte order, either unsigned int or #mdb_size_t.
+	 *	(lmdb expects 32-bit int <= size_t <= 32/64-bit mdb_size_t.)
 	 *  The keys must all be of the same size. */
 #define MDB_INTEGERKEY	0x08
 	/** with #MDB_DUPSORT, sorted dup items have fixed size */
@@ -1126,7 +1127,8 @@ int  mdb_txn_renew(MDB_txn *txn);
 	 *		keys must be unique and may have only a single data item.
 	 *	<li>#MDB_INTEGERKEY
 	 *		Keys are binary integers in native byte order, either unsigned int
-	 *		or size_t, and will be sorted as such.
+	 *		or #mdb_size_t, and will be sorted as such.
+	 *		(lmdb expects 32-bit int <= size_t <= 32/64-bit mdb_size_t.)
 	 *		The keys must all be of the same size.
 	 *	<li>#MDB_DUPFIXED
 	 *		This flag may only be used in combination with #MDB_DUPSORT. This option
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index ba5a39dd8b..6e1a3702c9 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1514,7 +1514,7 @@ static int mdb_reader_check0(MDB_env *env, int rlocked, int *dead);
 static MDB_cmp_func	mdb_cmp_memn, mdb_cmp_memnr, mdb_cmp_int, mdb_cmp_cint, mdb_cmp_long;
 /** @endcond */
 
-/** Compare two items pointing at size_t's of unknown alignment. */
+/** Compare two items pointing at '#mdb_size_t's of unknown alignment. */
 #ifdef MISALIGNED_OK
 # define mdb_cmp_clong mdb_cmp_long
 #else

From a43fcad8c65c2f4998a27d5596677dd81a42bd0f Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 23 Jul 2016 12:43:16 +0200
Subject: [PATCH 246/504] MDB_VL32: Match mdb_size_t type with format modifier.

When using format modifier "ll" or "I64", use the matching
type unsigned <long long / __int64> rather than uint64_t.
---
 libraries/liblmdb/lmdb.h | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index c21115d808..0b3d839ef1 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -167,6 +167,7 @@
 
 #include <sys/types.h>
 #include <inttypes.h>
+#include <limits.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -185,19 +186,24 @@ typedef	mode_t	mdb_mode_t;
 # define MDB_FMT_Z	"z"			/**< printf/scanf format modifier for size_t */
 #endif
 
-#ifdef MDB_VL32
-typedef uint64_t	mdb_size_t;
-#define MDB_SIZE_MAX UINT64_MAX
-#ifdef _WIN32
-# define MDB_FMT_Y	"I64"
-#else
-# define MDB_FMT_Y	"ll"
-#endif
-#define mdb_env_create	mdb_env_create_vl32	/**< Prevent mixing with non-VL32 builds */
-#else
+#if !defined(MDB_VL32) || SIZE_MAX > 0xffffffffU
 typedef size_t	mdb_size_t;
 # define MDB_SIZE_MAX	SIZE_MAX	/**< max #mdb_size_t */
 # define MDB_FMT_Y		MDB_FMT_Z	/**< Obsolescent, see #MDB_PRIz()/#MDB_SCNz() */
+/* TODO: For VL32, use uint64_t (trivial) and therefore PRI<c>64 (big patch) */
+#elif defined(_WIN32)
+typedef unsigned __int64 mdb_size_t;
+# define MDB_SIZE_MAX	_UI64_MAX
+# define MDB_FMT_Y		"I64"
+#elif defined(ULLONG_MAX)
+typedef unsigned long long	mdb_size_t;
+# define MDB_SIZE_MAX	ULLONG_MAX
+# define MDB_FMT_Y		"ll"
+#else
+# error "Found no acceptable integer type for mdb_size_t"
+#endif
+#ifdef MDB_VL32
+# define mdb_env_create	mdb_env_create_vl32	/**< Prevent mixing with non-VL32 builds */
 #endif
 
 /** #mdb_size_t printf formats, \b t = one of [diouxX] without quotes */

From 65d9791ada4672d2c7fb93a65e0ce7a1e4474658 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 23 Jul 2016 12:45:46 +0200
Subject: [PATCH 247/504] Refactor mdb_page_get()

---
 libraries/liblmdb/mdb.c | 39 ++++++++++++++++-----------------------
 1 file changed, 16 insertions(+), 23 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 6e1a3702c9..3a1dc6e867 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5957,9 +5957,6 @@ static int
 mdb_page_get(MDB_cursor *mc, pgno_t pgno, MDB_page **ret, int *lvl)
 {
 	MDB_txn *txn = mc->mc_txn;
-#ifndef MDB_VL32
-	MDB_env *env = txn->mt_env;
-#endif
 	MDB_page *p = NULL;
 	int level;
 
@@ -5978,14 +5975,7 @@ mdb_page_get(MDB_cursor *mc, pgno_t pgno, MDB_page **ret, int *lvl)
 				MDB_ID pn = pgno << 1;
 				x = mdb_midl_search(tx2->mt_spill_pgs, pn);
 				if (x <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[x] == pn) {
-#ifdef MDB_VL32
-					int rc = mdb_rpage_get(txn, pgno, &p);
-					if (rc)
-						return rc;
-#else
-					p = (MDB_page *)(env->me_map + env->me_psize * pgno);
-#endif
-					goto done;
+					goto mapped;
 				}
 			}
 			if (dl[0].mid) {
@@ -5999,23 +5989,26 @@ mdb_page_get(MDB_cursor *mc, pgno_t pgno, MDB_page **ret, int *lvl)
 		} while ((tx2 = tx2->mt_parent) != NULL);
 	}
 
-	if (pgno < txn->mt_next_pgno) {
-		level = 0;
-#ifdef MDB_VL32
-		{
-			int rc = mdb_rpage_get(txn, pgno, &p);
-			if (rc)
-				return rc;
-		}
-#else
-		p = (MDB_page *)(env->me_map + env->me_psize * pgno);
-#endif
-	} else {
+	if (pgno >= txn->mt_next_pgno) {
 		DPRINTF(("page %"Y"u not found", pgno));
 		txn->mt_flags |= MDB_TXN_ERROR;
 		return MDB_PAGE_NOTFOUND;
 	}
 
+	level = 0;
+
+mapped:
+	{
+#ifdef MDB_VL32
+		int rc = mdb_rpage_get(txn, pgno, &p);
+		if (rc)
+			return rc;
+#else
+		MDB_env *env = txn->mt_env;
+		p = (MDB_page *)(env->me_map + env->me_psize * pgno);
+#endif
+	}
+
 done:
 	*ret = p;
 	if (lvl)

From 4d47e89f4d4e4bacf2e58a420e3a2e84840fbe8e Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 2 Aug 2016 21:02:35 +0200
Subject: [PATCH 248/504] MDB_VL32: Switch to mdb_size_t formats PRIu64 & co

Drop macro Y=MDB_FMT_Y, add Yu/Yd=MDB_PRIy(). Replace
Y"d..." -> Yd"...", Y"u..." -> Yu"..." / MDB_SCNy(u)"...".
---
 libraries/liblmdb/lmdb.h     |  28 +++----
 libraries/liblmdb/mdb.c      | 141 +++++++++++++++++------------------
 libraries/liblmdb/mdb_dump.c |   4 +-
 libraries/liblmdb/mdb_load.c |  31 ++++----
 libraries/liblmdb/mdb_stat.c |  24 +++---
 5 files changed, 109 insertions(+), 119 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 0b3d839ef1..27821017ca 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -186,31 +186,21 @@ typedef	mode_t	mdb_mode_t;
 # define MDB_FMT_Z	"z"			/**< printf/scanf format modifier for size_t */
 #endif
 
-#if !defined(MDB_VL32) || SIZE_MAX > 0xffffffffU
+#ifndef MDB_VL32
 typedef size_t	mdb_size_t;
 # define MDB_SIZE_MAX	SIZE_MAX	/**< max #mdb_size_t */
-# define MDB_FMT_Y		MDB_FMT_Z	/**< Obsolescent, see #MDB_PRIz()/#MDB_SCNz() */
-/* TODO: For VL32, use uint64_t (trivial) and therefore PRI<c>64 (big patch) */
-#elif defined(_WIN32)
-typedef unsigned __int64 mdb_size_t;
-# define MDB_SIZE_MAX	_UI64_MAX
-# define MDB_FMT_Y		"I64"
-#elif defined(ULLONG_MAX)
-typedef unsigned long long	mdb_size_t;
-# define MDB_SIZE_MAX	ULLONG_MAX
-# define MDB_FMT_Y		"ll"
+/** #mdb_size_t printf formats, \b t = one of [diouxX] without quotes */
+# define MDB_PRIy(t)	MDB_FMT_Z #t
+/** #mdb_size_t scanf formats, \b t = one of [dioux] without quotes */
+# define MDB_SCNy(t)	MDB_FMT_Z #t
 #else
-# error "Found no acceptable integer type for mdb_size_t"
-#endif
-#ifdef MDB_VL32
+typedef uint64_t	mdb_size_t;
+# define MDB_SIZE_MAX	UINT64_MAX
+# define MDB_PRIy(t)	PRI##t##64
+# define MDB_SCNy(t)	SCN##t##64
 # define mdb_env_create	mdb_env_create_vl32	/**< Prevent mixing with non-VL32 builds */
 #endif
 
-/** #mdb_size_t printf formats, \b t = one of [diouxX] without quotes */
-#define MDB_PRIz(t)	MDB_FMT_Y #t
-/** #mdb_size_t scanf formats, \b t = one of [dioux] without quotes */
-#define MDB_SCNz(t)	MDB_FMT_Y #t
-
 /** An abstraction for a file handle.
  *	On POSIX systems file handles are small integers. On Windows
  *	they're opaque pointers.
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 3a1dc6e867..0d2889fd29 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -457,7 +457,8 @@ typedef pthread_mutex_t mdb_mutex_t[1], *mdb_mutexref_t;
 #endif
 
 #define	Z	MDB_FMT_Z	/**< printf/scanf format modifier for size_t */
-#define	Y	MDB_FMT_Y	/**< printf/scanf format modifier for #mdb_size_t */
+#define	Yu	MDB_PRIy(u)	/**< printf format for #mdb_size_t */
+#define	Yd	MDB_PRIy(d)	/**< printf format for "signed #mdb_size_t" */
 
 #if defined(_WIN32) || defined(MDB_USE_POSIX_SEM)
 #define MNAME_LEN	32
@@ -1711,20 +1712,20 @@ mdb_page_list(MDB_page *mp)
 	case P_LEAF|P_LEAF2:        type = "LEAF2 page";		break;
 	case P_LEAF|P_LEAF2|P_SUBP: type = "LEAF2 sub-page";	break;
 	case P_OVERFLOW:
-		fprintf(stderr, "Overflow page %"Y"u pages %u%s\n",
+		fprintf(stderr, "Overflow page %"Yu" pages %u%s\n",
 			pgno, mp->mp_pages, state);
 		return;
 	case P_META:
-		fprintf(stderr, "Meta-page %"Y"u txnid %"Y"u\n",
+		fprintf(stderr, "Meta-page %"Yu" txnid %"Yu"\n",
 			pgno, ((MDB_meta *)METADATA(mp))->mm_txnid);
 		return;
 	default:
-		fprintf(stderr, "Bad page %"Y"u flags 0x%u\n", pgno, mp->mp_flags);
+		fprintf(stderr, "Bad page %"Yu" flags 0x%u\n", pgno, mp->mp_flags);
 		return;
 	}
 
 	nkeys = NUMKEYS(mp);
-	fprintf(stderr, "%s %"Y"u numkeys %d%s\n", type, pgno, nkeys, state);
+	fprintf(stderr, "%s %"Yu" numkeys %d%s\n", type, pgno, nkeys, state);
 
 	for (i=0; i<nkeys; i++) {
 		if (IS_LEAF2(mp)) {	/* LEAF2 pages have no mp_ptrs[] or node headers */
@@ -1739,7 +1740,7 @@ mdb_page_list(MDB_page *mp)
 		key.mv_data = node->mn_data;
 		nsize = NODESIZE + key.mv_size;
 		if (IS_BRANCH(mp)) {
-			fprintf(stderr, "key %d: page %"Y"u, %s\n", i, NODEPGNO(node),
+			fprintf(stderr, "key %d: page %"Yu", %s\n", i, NODEPGNO(node),
 				DKEY(&key));
 			total += nsize;
 		} else {
@@ -1835,7 +1836,7 @@ static void mdb_audit(MDB_txn *txn)
 		}
 	}
 	if (freecount + count + NUM_METAS != txn->mt_next_pgno) {
-		fprintf(stderr, "audit: %"Y"u freecount: %"Y"u count: %"Y"u total: %"Y"u next_pgno: %"Y"u\n",
+		fprintf(stderr, "audit: %"Yu" freecount: %"Yu" count: %"Yu" total: %"Yu" next_pgno: %"Yu"\n",
 			txn->mt_txnid, freecount, count+NUM_METAS,
 			freecount+count+NUM_METAS, txn->mt_next_pgno);
 	}
@@ -2019,8 +2020,7 @@ mdb_page_loose(MDB_cursor *mc, MDB_page *mp)
 		}
 	}
 	if (loose) {
-		DPRINTF(("loosen db %d page %"Y"u", DDBI(mc),
-			mp->mp_pgno));
+		DPRINTF(("loosen db %d page %"Yu, DDBI(mc), mp->mp_pgno));
 		NEXT_LOOSE_PAGE(mp) = txn->mt_loose_pgs;
 		txn->mt_loose_pgs = mp;
 		txn->mt_loose_count++;
@@ -2317,8 +2317,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
 		np = txn->mt_loose_pgs;
 		txn->mt_loose_pgs = NEXT_LOOSE_PAGE(np);
 		txn->mt_loose_count--;
-		DPRINTF(("db %d use loose page %"Y"u", DDBI(mc),
-				np->mp_pgno));
+		DPRINTF(("db %d use loose page %"Yu, DDBI(mc), np->mp_pgno));
 		*mp = np;
 		return MDB_SUCCESS;
 	}
@@ -2420,10 +2419,10 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
 		}
 		env->me_pglast = last;
 #if (MDB_DEBUG) > 1
-		DPRINTF(("IDL read txn %"Y"u root %"Y"u num %u",
+		DPRINTF(("IDL read txn %"Yu" root %"Yu" num %u",
 			last, txn->mt_dbs[FREE_DBI].md_root, i));
 		for (j = i; j; j--)
-			DPRINTF(("IDL %"Y"u", idl[j]));
+			DPRINTF(("IDL %"Yu, idl[j]));
 #endif
 		/* Merge in descending sorted order */
 		mdb_midl_xmerge(mop, idl);
@@ -2593,7 +2592,7 @@ mdb_page_touch(MDB_cursor *mc)
 			(rc = mdb_page_alloc(mc, 1, &np)))
 			goto fail;
 		pgno = np->mp_pgno;
-		DPRINTF(("touched db %d page %"Y"u -> %"Y"u", DDBI(mc),
+		DPRINTF(("touched db %d page %"Yu" -> %"Yu, DDBI(mc),
 			mp->mp_pgno, pgno));
 		mdb_cassert(mc, mp->mp_pgno != pgno);
 		mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno);
@@ -2994,7 +2993,7 @@ mdb_txn_renew(MDB_txn *txn)
 
 	rc = mdb_txn_renew0(txn);
 	if (rc == MDB_SUCCESS) {
-		DPRINTF(("renew txn %"Y"u%c %p on mdbenv %p, root page %"Y"u",
+		DPRINTF(("renew txn %"Yu"%c %p on mdbenv %p, root page %"Yu,
 			txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w',
 			(void *)txn, (void *)txn->mt_env, txn->mt_dbs[MAIN_DBI].md_root));
 	}
@@ -3112,7 +3111,7 @@ renew:
 	} else {
 		txn->mt_flags |= flags;	/* could not change txn=me_txn0 earlier */
 		*ret = txn;
-		DPRINTF(("begin txn %"Y"u%c %p on mdbenv %p, root page %"Y"u",
+		DPRINTF(("begin txn %"Yu"%c %p on mdbenv %p, root page %"Yu,
 			txn->mt_txnid, (flags & MDB_RDONLY) ? 'r' : 'w',
 			(void *) txn, (void *) env, txn->mt_dbs[MAIN_DBI].md_root));
 	}
@@ -3179,7 +3178,7 @@ mdb_txn_end(MDB_txn *txn, unsigned mode)
 	/* Export or close DBI handles opened in this txn */
 	mdb_dbis_update(txn, mode & MDB_END_UPDATE);
 
-	DPRINTF(("%s txn %"Y"u%c %p on mdbenv %p, root page %"Y"u",
+	DPRINTF(("%s txn %"Yu"%c %p on mdbenv %p, root page %"Yu,
 		names[mode & MDB_END_OPMASK],
 		txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w',
 		(void *) txn, (void *)env, txn->mt_dbs[MAIN_DBI].md_root));
@@ -3380,10 +3379,10 @@ mdb_freelist_save(MDB_txn *txn)
 #if (MDB_DEBUG) > 1
 			{
 				unsigned int i = free_pgs[0];
-				DPRINTF(("IDL write txn %"Y"u root %"Y"u num %u",
+				DPRINTF(("IDL write txn %"Yu" root %"Yu" num %u",
 					txn->mt_txnid, txn->mt_dbs[FREE_DBI].md_root, i));
 				for (; i; i--)
-					DPRINTF(("IDL %"Y"u", free_pgs[i]));
+					DPRINTF(("IDL %"Yu, free_pgs[i]));
 			}
 #endif
 			continue;
@@ -3551,7 +3550,7 @@ mdb_page_flush(MDB_txn *txn, int keep)
 		 * the write offset, to at least save the overhead of a Seek
 		 * system call.
 		 */
-		DPRINTF(("committing page %"Y"u", pgno));
+		DPRINTF(("committing page %"Yu, pgno));
 		memset(&ov, 0, sizeof(ov));
 		ov.Offset = pos & 0xffffffff;
 		ov.OffsetHigh = pos >> 16 >> 16;
@@ -3602,7 +3601,7 @@ retry_seek:
 			wpos = pos;
 			wsize = 0;
 		}
-		DPRINTF(("committing page %"Y"u", pgno));
+		DPRINTF(("committing page %"Yu, pgno));
 		next_pos = pos + size;
 		iov[n].iov_len = size;
 		iov[n].iov_base = (char *)dp;
@@ -3815,7 +3814,7 @@ mdb_txn_commit(MDB_txn *txn)
 		!(txn->mt_flags & (MDB_TXN_DIRTY|MDB_TXN_SPILLS)))
 		goto done;
 
-	DPRINTF(("committing txn %"Y"u %p on mdbenv %p, root page %"Y"u",
+	DPRINTF(("committing txn %"Yu" %p on mdbenv %p, root page %"Yu,
 	    txn->mt_txnid, (void*)txn, (void*)env, txn->mt_dbs[MAIN_DBI].md_root));
 
 	/* Update DB root pointers */
@@ -3913,7 +3912,7 @@ mdb_env_read_header(MDB_env *env, MDB_meta *meta)
 		p = (MDB_page *)&pbuf;
 
 		if (!F_ISSET(p->mp_flags, P_META)) {
-			DPRINTF(("page %"Y"u not a meta page", p->mp_pgno));
+			DPRINTF(("page %"Yu" not a meta page", p->mp_pgno));
 			return MDB_INVALID;
 		}
 
@@ -4025,7 +4024,7 @@ mdb_env_write_meta(MDB_txn *txn)
 #endif
 
 	toggle = txn->mt_txnid & 1;
-	DPRINTF(("writing meta page %d for root page %"Y"u",
+	DPRINTF(("writing meta page %d for root page %"Yu,
 		toggle, txn->mt_dbs[MAIN_DBI].md_root));
 
 	env = txn->mt_env;
@@ -4519,13 +4518,13 @@ mdb_env_open2(MDB_env *env)
 
 		DPRINTF(("opened database version %u, pagesize %u",
 			meta->mm_version, env->me_psize));
-		DPRINTF(("using meta page %d",    (int) (meta->mm_txnid & 1)));
-		DPRINTF(("depth: %u",             db->md_depth));
-		DPRINTF(("entries: %"Y"u",        db->md_entries));
-		DPRINTF(("branch pages: %"Y"u",   db->md_branch_pages));
-		DPRINTF(("leaf pages: %"Y"u",     db->md_leaf_pages));
-		DPRINTF(("overflow pages: %"Y"u", db->md_overflow_pages));
-		DPRINTF(("root: %"Y"u",           db->md_root));
+		DPRINTF(("using meta page %d",  (int) (meta->mm_txnid & 1)));
+		DPRINTF(("depth: %u",           db->md_depth));
+		DPRINTF(("entries: %"Yu,        db->md_entries));
+		DPRINTF(("branch pages: %"Yu,   db->md_branch_pages));
+		DPRINTF(("leaf pages: %"Yu,     db->md_leaf_pages));
+		DPRINTF(("overflow pages: %"Yu, db->md_overflow_pages));
+		DPRINTF(("root: %"Yu,           db->md_root));
 	}
 #endif
 
@@ -5531,7 +5530,7 @@ mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp)
 
 	nkeys = NUMKEYS(mp);
 
-	DPRINTF(("searching %u keys in %s %spage %"Y"u",
+	DPRINTF(("searching %u keys in %s %spage %"Yu,
 	    nkeys, IS_LEAF(mp) ? "leaf" : "branch", IS_SUBP(mp) ? "sub-" : "",
 	    mdb_dbg_pgno(mp)));
 
@@ -5579,7 +5578,7 @@ mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp)
 				DPRINTF(("found leaf index %u [%s], rc = %i",
 				    i, DKEY(&nodekey), rc));
 			else
-				DPRINTF(("found branch index %u [%s -> %"Y"u], rc = %i",
+				DPRINTF(("found branch index %u [%s -> %"Yu"], rc = %i",
 				    i, DKEY(&nodekey), NODEPGNO(node), rc));
 #endif
 			if (rc == 0)
@@ -5627,7 +5626,7 @@ static void
 mdb_cursor_pop(MDB_cursor *mc)
 {
 	if (mc->mc_snum) {
-		DPRINTF(("popping page %"Y"u off db %d cursor %p",
+		DPRINTF(("popping page %"Yu" off db %d cursor %p",
 			mc->mc_pg[mc->mc_top]->mp_pgno, DDBI(mc), (void *) mc));
 
 		mc->mc_snum--;
@@ -5643,7 +5642,7 @@ mdb_cursor_pop(MDB_cursor *mc)
 static int
 mdb_cursor_push(MDB_cursor *mc, MDB_page *mp)
 {
-	DPRINTF(("pushing page %"Y"u on db %d cursor %p", mp->mp_pgno,
+	DPRINTF(("pushing page %"Yu" on db %d cursor %p", mp->mp_pgno,
 		DDBI(mc), (void *) mc));
 
 	if (mc->mc_snum >= CURSOR_STACK) {
@@ -5990,7 +5989,7 @@ mdb_page_get(MDB_cursor *mc, pgno_t pgno, MDB_page **ret, int *lvl)
 	}
 
 	if (pgno >= txn->mt_next_pgno) {
-		DPRINTF(("page %"Y"u not found", pgno));
+		DPRINTF(("page %"Yu" not found", pgno));
 		txn->mt_flags |= MDB_TXN_ERROR;
 		return MDB_PAGE_NOTFOUND;
 	}
@@ -6030,13 +6029,13 @@ mdb_page_search_root(MDB_cursor *mc, MDB_val *key, int flags)
 		MDB_node	*node;
 		indx_t		i;
 
-		DPRINTF(("branch page %"Y"u has %u keys", mp->mp_pgno, NUMKEYS(mp)));
+		DPRINTF(("branch page %"Yu" has %u keys", mp->mp_pgno, NUMKEYS(mp)));
 		/* Don't assert on branch pages in the FreeDB. We can get here
 		 * while in the process of rebalancing a FreeDB branch page; we must
 		 * let that proceed. ITS#8336
 		 */
 		mdb_cassert(mc, !mc->mc_dbi || NUMKEYS(mp) > 1);
-		DPRINTF(("found index 0 to page %"Y"u", NODEPGNO(NODEPTR(mp, 0))));
+		DPRINTF(("found index 0 to page %"Yu, NODEPGNO(NODEPTR(mp, 0))));
 
 		if (flags & (MDB_PS_FIRST|MDB_PS_LAST)) {
 			i = 0;
@@ -6081,7 +6080,7 @@ mdb_page_search_root(MDB_cursor *mc, MDB_val *key, int flags)
 		return MDB_CORRUPTED;
 	}
 
-	DPRINTF(("found leaf page %"Y"u for key [%s]", mp->mp_pgno,
+	DPRINTF(("found leaf page %"Yu" for key [%s]", mp->mp_pgno,
 	    key ? DKEY(key) : "null"));
 	mc->mc_flags |= C_INITIALIZED;
 	mc->mc_flags &= ~C_EOF;
@@ -6196,7 +6195,7 @@ mdb_page_search(MDB_cursor *mc, MDB_val *key, int flags)
 	mc->mc_snum = 1;
 	mc->mc_top = 0;
 
-	DPRINTF(("db %d root page %"Y"u has flags 0x%X",
+	DPRINTF(("db %d root page %"Yu" has flags 0x%X",
 		DDBI(mc), root, mc->mc_pg[0]->mp_flags));
 
 	if (flags & MDB_PS_MODIFY) {
@@ -6221,7 +6220,7 @@ mdb_ovpage_free(MDB_cursor *mc, MDB_page *mp)
 	MDB_ID pn = pg << 1;
 	int rc;
 
-	DPRINTF(("free ov page %"Y"u (%d)", pg, ovpages));
+	DPRINTF(("free ov page %"Yu" (%d)", pg, ovpages));
 	/* If the page is dirty or on the spill list we just acquired it,
 	 * so we should give it back to our current free list, if any.
 	 * Otherwise put it onto the list of pages we freed in this txn.
@@ -6314,7 +6313,7 @@ mdb_node_read(MDB_cursor *mc, MDB_node *leaf, MDB_val *data)
 	data->mv_size = NODEDSZ(leaf);
 	memcpy(&pgno, NODEDATA(leaf), sizeof(pgno));
 	if ((rc = mdb_page_get(mc, pgno, &omp, NULL)) != 0) {
-		DPRINTF(("read overflow page %"Y"u failed", pgno));
+		DPRINTF(("read overflow page %"Yu" failed", pgno));
 		return rc;
 	}
 	data->mv_data = METADATA(omp);
@@ -6375,7 +6374,7 @@ mdb_cursor_sibling(MDB_cursor *mc, int move_right)
 	op = mc->mc_pg[mc->mc_top];
 #endif
 	mdb_cursor_pop(mc);
-	DPRINTF(("parent page is page %"Y"u, index %u",
+	DPRINTF(("parent page is page %"Yu", index %u",
 		mc->mc_pg[mc->mc_top]->mp_pgno, mc->mc_ki[mc->mc_top]));
 
 	if (move_right ? (mc->mc_ki[mc->mc_top] + 1u >= NUMKEYS(mc->mc_pg[mc->mc_top]))
@@ -6452,7 +6451,7 @@ mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
 		}
 	}
 
-	DPRINTF(("cursor_next: top page is %"Y"u in cursor %p",
+	DPRINTF(("cursor_next: top page is %"Yu" in cursor %p",
 		mdb_dbg_pgno(mp), (void *) mc));
 	if (mc->mc_flags & C_DEL) {
 		mc->mc_flags ^= C_DEL;
@@ -6466,12 +6465,12 @@ mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
 			return rc;
 		}
 		mp = mc->mc_pg[mc->mc_top];
-		DPRINTF(("next page is %"Y"u, key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top]));
+		DPRINTF(("next page is %"Yu", key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top]));
 	} else
 		mc->mc_ki[mc->mc_top]++;
 
 skip:
-	DPRINTF(("==> cursor points to page %"Y"u with %u keys, key index %u",
+	DPRINTF(("==> cursor points to page %"Yu" with %u keys, key index %u",
 	    mdb_dbg_pgno(mp), NUMKEYS(mp), mc->mc_ki[mc->mc_top]));
 
 	if (IS_LEAF2(mp)) {
@@ -6541,7 +6540,7 @@ mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
 		}
 	}
 
-	DPRINTF(("cursor_prev: top page is %"Y"u in cursor %p",
+	DPRINTF(("cursor_prev: top page is %"Yu" in cursor %p",
 		mdb_dbg_pgno(mp), (void *) mc));
 
 	mc->mc_flags &= ~(C_EOF|C_DEL);
@@ -6553,13 +6552,13 @@ mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
 		}
 		mp = mc->mc_pg[mc->mc_top];
 		mc->mc_ki[mc->mc_top] = NUMKEYS(mp) - 1;
-		DPRINTF(("prev page is %"Y"u, key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top]));
+		DPRINTF(("prev page is %"Yu", key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top]));
 	} else
 		mc->mc_ki[mc->mc_top]--;
 
 	mc->mc_flags &= ~C_EOF;
 
-	DPRINTF(("==> cursor points to page %"Y"u with %u keys, key index %u",
+	DPRINTF(("==> cursor points to page %"Yu" with %u keys, key index %u",
 	    mdb_dbg_pgno(mp), NUMKEYS(mp), mc->mc_ki[mc->mc_top]));
 
 	if (IS_LEAF2(mp)) {
@@ -7753,7 +7752,7 @@ mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp)
 
 	if ((rc = mdb_page_alloc(mc, num, &np)))
 		return rc;
-	DPRINTF(("allocated new mpage %"Y"u, page size %u",
+	DPRINTF(("allocated new mpage %"Yu", page size %u",
 	    np->mp_pgno, mc->mc_txn->mt_env->me_psize));
 	np->mp_flags = flags | P_DIRTY;
 	np->mp_lower = (PAGEHDRSZ-PAGEBASE);
@@ -7853,7 +7852,7 @@ mdb_node_add(MDB_cursor *mc, indx_t indx,
 
 	mdb_cassert(mc, mp->mp_upper >= mp->mp_lower);
 
-	DPRINTF(("add to %s %spage %"Y"u index %i, data size %"Z"u key size %"Z"u [%s]",
+	DPRINTF(("add to %s %spage %"Yu" index %i, data size %"Z"u key size %"Z"u [%s]",
 	    IS_LEAF(mp) ? "leaf" : "branch",
 		IS_SUBP(mp) ? "sub-" : "",
 		mdb_dbg_pgno(mp), indx, data ? data->mv_size : 0,
@@ -7894,7 +7893,7 @@ mdb_node_add(MDB_cursor *mc, indx_t indx,
 				goto full;
 			if ((rc = mdb_page_new(mc, P_OVERFLOW, ovpages, &ofp)))
 				return rc;
-			DPRINTF(("allocated overflow page %"Y"u", ofp->mp_pgno));
+			DPRINTF(("allocated overflow page %"Yu, ofp->mp_pgno));
 			flags |= F_BIGDATA;
 			goto update;
 		} else {
@@ -7951,7 +7950,7 @@ update:
 	return MDB_SUCCESS;
 
 full:
-	DPRINTF(("not enough room in page %"Y"u, got %u ptrs",
+	DPRINTF(("not enough room in page %"Yu", got %u ptrs",
 		mdb_dbg_pgno(mp), NUMKEYS(mp)));
 	DPRINTF(("upper-lower = %u - %u = %"Z"d", mp->mp_upper,mp->mp_lower,room));
 	DPRINTF(("node size = %"Z"u", node_size));
@@ -7974,7 +7973,7 @@ mdb_node_del(MDB_cursor *mc, int ksize)
 	MDB_node	*node;
 	char		*base;
 
-	DPRINTF(("delete node %u on %s page %"Y"u", indx,
+	DPRINTF(("delete node %u on %s page %"Yu, indx,
 	    IS_LEAF(mp) ? "leaf" : "branch", mdb_dbg_pgno(mp)));
 	numkeys = NUMKEYS(mp);
 	mdb_cassert(mc, indx < numkeys);
@@ -8131,7 +8130,7 @@ mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node)
 				mx->mx_db.md_flags |= MDB_INTEGERKEY;
 		}
 	}
-	DPRINTF(("Sub-db -%u root page %"Y"u", mx->mx_cursor.mc_dbi,
+	DPRINTF(("Sub-db -%u root page %"Yu, mx->mx_cursor.mc_dbi,
 		mx->mx_db.md_root));
 	mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DIRTY; /* DB_DIRTY guides mdb_cursor_touch */
 	if (NEED_CMP_CLONG(mx->mx_dbx.md_cmp, mx->mx_db.md_pad))
@@ -8166,7 +8165,7 @@ mdb_xcursor_init2(MDB_cursor *mc, MDB_xcursor *src_mx, int new_dupdata)
 	}
 	mx->mx_db = src_mx->mx_db;
 	mx->mx_cursor.mc_pg[0] = src_mx->mx_cursor.mc_pg[0];
-	DPRINTF(("Sub-db -%u root page %"Y"u", mx->mx_cursor.mc_dbi,
+	DPRINTF(("Sub-db -%u root page %"Yu, mx->mx_cursor.mc_dbi,
 		mx->mx_db.md_root));
 }
 
@@ -8336,7 +8335,7 @@ mdb_update_key(MDB_cursor *mc, MDB_val *key)
 		char kbuf2[DKBUF_MAXKEYSIZE*2+1];
 		k2.mv_data = NODEKEY(node);
 		k2.mv_size = node->mn_ksize;
-		DPRINTF(("update key %u (ofs %u) [%s] to [%s] on page %"Y"u",
+		DPRINTF(("update key %u (ofs %u) [%s] to [%s] on page %"Yu,
 			indx, ptr,
 			mdb_dkey(&k2, kbuf2),
 			DKEY(key),
@@ -8484,7 +8483,7 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft)
 			return rc;
 	}
 
-	DPRINTF(("moving %s node %u [%s] on page %"Y"u to node %u on page %"Y"u",
+	DPRINTF(("moving %s node %u [%s] on page %"Yu" to node %u on page %"Yu,
 	    IS_LEAF(csrc->mc_pg[csrc->mc_top]) ? "leaf" : "branch",
 	    csrc->mc_ki[csrc->mc_top],
 		DKEY(&key),
@@ -8578,7 +8577,7 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft)
 				key.mv_size = NODEKSZ(srcnode);
 				key.mv_data = NODEKEY(srcnode);
 			}
-			DPRINTF(("update separator for source page %"Y"u to [%s]",
+			DPRINTF(("update separator for source page %"Yu" to [%s]",
 				csrc->mc_pg[csrc->mc_top]->mp_pgno, DKEY(&key)));
 			mdb_cursor_copy(csrc, &mn);
 			mn.mc_snum--;
@@ -8609,7 +8608,7 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft)
 				key.mv_size = NODEKSZ(srcnode);
 				key.mv_data = NODEKEY(srcnode);
 			}
-			DPRINTF(("update separator for destination page %"Y"u to [%s]",
+			DPRINTF(("update separator for destination page %"Yu" to [%s]",
 				cdst->mc_pg[cdst->mc_top]->mp_pgno, DKEY(&key)));
 			mdb_cursor_copy(cdst, &mn);
 			mn.mc_snum--;
@@ -8655,7 +8654,7 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst)
 	psrc = csrc->mc_pg[csrc->mc_top];
 	pdst = cdst->mc_pg[cdst->mc_top];
 
-	DPRINTF(("merging page %"Y"u into %"Y"u", psrc->mp_pgno, pdst->mp_pgno));
+	DPRINTF(("merging page %"Yu" into %"Yu, psrc->mp_pgno, pdst->mp_pgno));
 
 	mdb_cassert(csrc, csrc->mc_snum > 1);	/* can't merge root page */
 	mdb_cassert(csrc, cdst->mc_snum > 1);
@@ -8712,7 +8711,7 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst)
 		}
 	}
 
-	DPRINTF(("dst page %"Y"u now has %u keys (%.1f%% filled)",
+	DPRINTF(("dst page %"Yu" now has %u keys (%.1f%% filled)",
 	    pdst->mp_pgno, NUMKEYS(pdst),
 		(float)PAGEFILL(cdst->mc_txn->mt_env, pdst) / 10));
 
@@ -8829,14 +8828,14 @@ mdb_rebalance(MDB_cursor *mc)
 		minkeys = 1;
 		thresh = FILL_THRESHOLD;
 	}
-	DPRINTF(("rebalancing %s page %"Y"u (has %u keys, %.1f%% full)",
+	DPRINTF(("rebalancing %s page %"Yu" (has %u keys, %.1f%% full)",
 	    IS_LEAF(mc->mc_pg[mc->mc_top]) ? "leaf" : "branch",
 	    mdb_dbg_pgno(mc->mc_pg[mc->mc_top]), NUMKEYS(mc->mc_pg[mc->mc_top]),
 		(float)PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) / 10));
 
 	if (PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) >= thresh &&
 		NUMKEYS(mc->mc_pg[mc->mc_top]) >= minkeys) {
-		DPRINTF(("no need to rebalance page %"Y"u, above fill threshold",
+		DPRINTF(("no need to rebalance page %"Yu", above fill threshold",
 		    mdb_dbg_pgno(mc->mc_pg[mc->mc_top])));
 		return MDB_SUCCESS;
 	}
@@ -8965,7 +8964,7 @@ mdb_rebalance(MDB_cursor *mc)
 		fromleft = 1;
 	}
 
-	DPRINTF(("found neighbor page %"Y"u (%u keys, %.1f%% full)",
+	DPRINTF(("found neighbor page %"Yu" (%u keys, %.1f%% full)",
 	    mn.mc_pg[mn.mc_top]->mp_pgno, NUMKEYS(mn.mc_pg[mn.mc_top]),
 		(float)PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) / 10));
 
@@ -9175,7 +9174,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 	newindx = mc->mc_ki[mc->mc_top];
 	nkeys = NUMKEYS(mp);
 
-	DPRINTF(("-----> splitting %s page %"Y"u and adding [%s] at index %i/%i",
+	DPRINTF(("-----> splitting %s page %"Yu" and adding [%s] at index %i/%i",
 	    IS_LEAF(mp) ? "leaf" : "branch", mp->mp_pgno,
 	    DKEY(newkey), mc->mc_ki[mc->mc_top], nkeys));
 
@@ -9183,7 +9182,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 	if ((rc = mdb_page_new(mc, mp->mp_flags, 1, &rp)))
 		return rc;
 	rp->mp_pad = mp->mp_pad;
-	DPRINTF(("new right sibling: page %"Y"u", rp->mp_pgno));
+	DPRINTF(("new right sibling: page %"Yu, rp->mp_pgno));
 
 	/* Usually when splitting the root page, the cursor
 	 * height is 1. But when called from mdb_update_key,
@@ -9201,7 +9200,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 		mc->mc_pg[0] = pp;
 		mc->mc_ki[0] = 0;
 		mc->mc_db->md_root = pp->mp_pgno;
-		DPRINTF(("root split! new root = %"Y"u", pp->mp_pgno));
+		DPRINTF(("root split! new root = %"Yu, pp->mp_pgno));
 		new_root = mc->mc_db->md_depth++;
 
 		/* Add left (implicit) pointer. */
@@ -9218,7 +9217,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 		ptop = 0;
 	} else {
 		ptop = mc->mc_top-1;
-		DPRINTF(("parent branch page is %"Y"u", mc->mc_pg[ptop]->mp_pgno));
+		DPRINTF(("parent branch page is %"Yu, mc->mc_pg[ptop]->mp_pgno));
 	}
 
 	mdb_cursor_copy(mc, &mn);
@@ -10694,7 +10693,7 @@ mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx)
 		if (mr[i].mr_pid) {
 			txnid_t	txnid = mr[i].mr_txnid;
 			sprintf(buf, txnid == (txnid_t)-1 ?
-				"%10d %"Z"x -\n" : "%10d %"Z"x %"Y"u\n",
+				"%10d %"Z"x -\n" : "%10d %"Z"x %"Yu"\n",
 				(int)mr[i].mr_pid, (size_t)mr[i].mr_tid, txnid);
 			if (first) {
 				first = 0;
@@ -10799,7 +10798,7 @@ mdb_reader_check0(MDB_env *env, int rlocked, int *dead)
 					}
 					for (; j<rdrs; j++)
 							if (mr[j].mr_pid == pid) {
-								DPRINTF(("clear stale reader pid %u txn %"Y"d",
+								DPRINTF(("clear stale reader pid %u txn %"Yd,
 									(unsigned) pid, mr[j].mr_txnid));
 								mr[j].mr_pid = 0;
 								count++;
diff --git a/libraries/liblmdb/mdb_dump.c b/libraries/liblmdb/mdb_dump.c
index 538b243950..13084a22ec 100644
--- a/libraries/liblmdb/mdb_dump.c
+++ b/libraries/liblmdb/mdb_dump.c
@@ -20,7 +20,7 @@
 #include <signal.h>
 #include "lmdb.h"
 
-#define Y	MDB_FMT_Y
+#define Yu	MDB_PRIy(u)
 
 #define PRINT	1
 static int mode;
@@ -111,7 +111,7 @@ static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name)
 	if (name)
 		printf("database=%s\n", name);
 	printf("type=btree\n");
-	printf("mapsize=%" Y "u\n", info.me_mapsize);
+	printf("mapsize=%"Yu"\n", info.me_mapsize);
 	if (info.me_mapaddr)
 		printf("mapaddr=%p\n", info.me_mapaddr);
 	printf("maxreaders=%u\n", info.me_maxreaders);
diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c
index 3b55a946d1..73dfe8cfd9 100644
--- a/libraries/liblmdb/mdb_load.c
+++ b/libraries/liblmdb/mdb_load.c
@@ -38,7 +38,7 @@ static MDB_envinfo info;
 
 static MDB_val kbuf, dbuf;
 
-#define Y	MDB_FMT_Y
+#define Yu	MDB_PRIy(u)
 
 #define STRLENOF(s)	(sizeof(s)-1)
 
@@ -69,7 +69,7 @@ static void readhdr(void)
 		if (!strncmp(dbuf.mv_data, "VERSION=", STRLENOF("VERSION="))) {
 			version=atoi((char *)dbuf.mv_data+STRLENOF("VERSION="));
 			if (version > 3) {
-				fprintf(stderr, "%s: line %" Y "u: unsupported VERSION %d\n",
+				fprintf(stderr, "%s: line %"Yu": unsupported VERSION %d\n",
 					prog, lineno, version);
 				exit(EXIT_FAILURE);
 			}
@@ -79,7 +79,7 @@ static void readhdr(void)
 			if (!strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "print", STRLENOF("print")))
 				mode |= PRINT;
 			else if (strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "bytevalue", STRLENOF("bytevalue"))) {
-				fprintf(stderr, "%s: line %" Y "u: unsupported FORMAT %s\n",
+				fprintf(stderr, "%s: line %"Yu": unsupported FORMAT %s\n",
 					prog, lineno, (char *)dbuf.mv_data+STRLENOF("FORMAT="));
 				exit(EXIT_FAILURE);
 			}
@@ -90,7 +90,7 @@ static void readhdr(void)
 			subname = strdup((char *)dbuf.mv_data+STRLENOF("database="));
 		} else if (!strncmp(dbuf.mv_data, "type=", STRLENOF("type="))) {
 			if (strncmp((char *)dbuf.mv_data+STRLENOF("type="), "btree", STRLENOF("btree")))  {
-				fprintf(stderr, "%s: line %" Y "u: unsupported type %s\n",
+				fprintf(stderr, "%s: line %"Yu": unsupported type %s\n",
 					prog, lineno, (char *)dbuf.mv_data+STRLENOF("type="));
 				exit(EXIT_FAILURE);
 			}
@@ -100,7 +100,7 @@ static void readhdr(void)
 			if (ptr) *ptr = '\0';
 			i = sscanf((char *)dbuf.mv_data+STRLENOF("mapaddr="), "%p", &info.me_mapaddr);
 			if (i != 1) {
-				fprintf(stderr, "%s: line %" Y "u: invalid mapaddr %s\n",
+				fprintf(stderr, "%s: line %"Yu": invalid mapaddr %s\n",
 					prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapaddr="));
 				exit(EXIT_FAILURE);
 			}
@@ -108,9 +108,10 @@ static void readhdr(void)
 			int i;
 			ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
 			if (ptr) *ptr = '\0';
-			i = sscanf((char *)dbuf.mv_data+STRLENOF("mapsize="), "%" Y "u", &info.me_mapsize);
+			i = sscanf((char *)dbuf.mv_data+STRLENOF("mapsize="),
+				"%" MDB_SCNy(u), &info.me_mapsize);
 			if (i != 1) {
-				fprintf(stderr, "%s: line %" Y "u: invalid mapsize %s\n",
+				fprintf(stderr, "%s: line %"Yu": invalid mapsize %s\n",
 					prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapsize="));
 				exit(EXIT_FAILURE);
 			}
@@ -120,7 +121,7 @@ static void readhdr(void)
 			if (ptr) *ptr = '\0';
 			i = sscanf((char *)dbuf.mv_data+STRLENOF("maxreaders="), "%u", &info.me_maxreaders);
 			if (i != 1) {
-				fprintf(stderr, "%s: line %" Y "u: invalid maxreaders %s\n",
+				fprintf(stderr, "%s: line %"Yu": invalid maxreaders %s\n",
 					prog, lineno, (char *)dbuf.mv_data+STRLENOF("maxreaders="));
 				exit(EXIT_FAILURE);
 			}
@@ -136,12 +137,12 @@ static void readhdr(void)
 			if (!dbflags[i].bit) {
 				ptr = memchr(dbuf.mv_data, '=', dbuf.mv_size);
 				if (!ptr) {
-					fprintf(stderr, "%s: line %" Y "u: unexpected format\n",
+					fprintf(stderr, "%s: line %"Yu": unexpected format\n",
 						prog, lineno);
 					exit(EXIT_FAILURE);
 				} else {
 					*ptr = '\0';
-					fprintf(stderr, "%s: line %" Y "u: unrecognized keyword ignored: %s\n",
+					fprintf(stderr, "%s: line %"Yu": unrecognized keyword ignored: %s\n",
 						prog, lineno, (char *)dbuf.mv_data);
 				}
 			}
@@ -151,7 +152,7 @@ static void readhdr(void)
 
 static void badend(void)
 {
-	fprintf(stderr, "%s: line %" Y "u: unexpected end of input\n",
+	fprintf(stderr, "%s: line %"Yu": unexpected end of input\n",
 		prog, lineno);
 }
 
@@ -209,7 +210,7 @@ badend:
 		buf->mv_data = realloc(buf->mv_data, buf->mv_size*2);
 		if (!buf->mv_data) {
 			Eof = 1;
-			fprintf(stderr, "%s: line %" Y "u: out of memory, line too long\n",
+			fprintf(stderr, "%s: line %"Yu": out of memory, line too long\n",
 				prog, lineno);
 			return EOF;
 		}
@@ -401,7 +402,7 @@ int main(int argc, char *argv[])
 
 			rc = readline(&data, &dbuf);
 			if (rc) {
-				fprintf(stderr, "%s: line %" Y "u: failed to read key value\n", prog, lineno);
+				fprintf(stderr, "%s: line %"Yu": failed to read key value\n", prog, lineno);
 				goto txn_abort;
 			}
 
@@ -416,7 +417,7 @@ int main(int argc, char *argv[])
 			if (batch == 100) {
 				rc = mdb_txn_commit(txn);
 				if (rc) {
-					fprintf(stderr, "%s: line %" Y "u: txn_commit: %s\n",
+					fprintf(stderr, "%s: line %"Yu": txn_commit: %s\n",
 						prog, lineno, mdb_strerror(rc));
 					goto env_close;
 				}
@@ -436,7 +437,7 @@ int main(int argc, char *argv[])
 		rc = mdb_txn_commit(txn);
 		txn = NULL;
 		if (rc) {
-			fprintf(stderr, "%s: line %" Y "u: txn_commit: %s\n",
+			fprintf(stderr, "%s: line %"Yu": txn_commit: %s\n",
 				prog, lineno, mdb_strerror(rc));
 			goto env_close;
 		}
diff --git a/libraries/liblmdb/mdb_stat.c b/libraries/liblmdb/mdb_stat.c
index 5c57072db4..0343a6537d 100644
--- a/libraries/liblmdb/mdb_stat.c
+++ b/libraries/liblmdb/mdb_stat.c
@@ -18,7 +18,7 @@
 #include "lmdb.h"
 
 #define Z	MDB_FMT_Z
-#define Y	MDB_FMT_Y
+#define Yu	MDB_PRIy(u)
 
 static void prstat(MDB_stat *ms)
 {
@@ -26,10 +26,10 @@ static void prstat(MDB_stat *ms)
 	printf("  Page size: %u\n", ms->ms_psize);
 #endif
 	printf("  Tree depth: %u\n", ms->ms_depth);
-	printf("  Branch pages: %"Y"u\n", ms->ms_branch_pages);
-	printf("  Leaf pages: %"Y"u\n", ms->ms_leaf_pages);
-	printf("  Overflow pages: %"Y"u\n", ms->ms_overflow_pages);
-	printf("  Entries: %"Y"u\n", ms->ms_entries);
+	printf("  Branch pages: %"Yu"\n",   ms->ms_branch_pages);
+	printf("  Leaf pages: %"Yu"\n",     ms->ms_leaf_pages);
+	printf("  Overflow pages: %"Yu"\n", ms->ms_overflow_pages);
+	printf("  Entries: %"Yu"\n",        ms->ms_entries);
 }
 
 static void usage(char *prog)
@@ -122,11 +122,11 @@ int main(int argc, char *argv[])
 		(void)mdb_env_info(env, &mei);
 		printf("Environment Info\n");
 		printf("  Map address: %p\n", mei.me_mapaddr);
-		printf("  Map size: %"Y"u\n", mei.me_mapsize);
+		printf("  Map size: %"Yu"\n", mei.me_mapsize);
 		printf("  Page size: %u\n", mst.ms_psize);
-		printf("  Max pages: %"Y"u\n", mei.me_mapsize / mst.ms_psize);
-		printf("  Number of pages used: %"Y"u\n", mei.me_last_pgno+1);
-		printf("  Last transaction ID: %"Y"u\n", mei.me_last_txnid);
+		printf("  Max pages: %"Yu"\n", mei.me_mapsize / mst.ms_psize);
+		printf("  Number of pages used: %"Yu"\n", mei.me_last_pgno+1);
+		printf("  Last transaction ID: %"Yu"\n", mei.me_last_txnid);
 		printf("  Max readers: %u\n", mei.me_maxreaders);
 		printf("  Number of readers used: %u\n", mei.me_numreaders);
 	}
@@ -184,20 +184,20 @@ int main(int argc, char *argv[])
 					pg += span;
 					for (; i >= span && iptr[i-span] == pg; span++, pg++) ;
 				}
-				printf("    Transaction %"Y"u, %"Z"d pages, maxspan %"Z"d%s\n",
+				printf("    Transaction %"Yu", %"Z"d pages, maxspan %"Z"d%s\n",
 					*(mdb_size_t *)key.mv_data, j, span, bad);
 				if (freinfo > 2) {
 					for (--j; j >= 0; ) {
 						pg = iptr[j];
 						for (span=1; --j >= 0 && iptr[j] == pg+span; span++) ;
-						printf(span>1 ? "     %9"Y"u[%"Z"d]\n" : "     %9"Y"u\n",
+						printf(span>1 ? "     %9"Yu"[%"Z"d]\n" : "     %9"Yu"\n",
 							pg, span);
 					}
 				}
 			}
 		}
 		mdb_cursor_close(cursor);
-		printf("  Free pages: %"Y"u\n", pages);
+		printf("  Free pages: %"Yu"\n", pages);
 	}
 
 	rc = mdb_open(txn, subname, 0, &dbi);

From 58b0ce50b3a4d76dd0f5c1ed780a75db54549558 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 2 Aug 2016 21:04:17 +0200
Subject: [PATCH 249/504] mdb_size comments

---
 libraries/liblmdb/lmdb.h | 5 +++++
 libraries/liblmdb/mdb.c  | 4 ++--
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 27821017ca..30d586203a 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -187,6 +187,11 @@ typedef	mode_t	mdb_mode_t;
 #endif
 
 #ifndef MDB_VL32
+/** Unsigned type used for mapsize, entry counts and page/transaction IDs.
+ *
+ *	It is normally size_t, hence the name. Defining MDB_VL32 makes it
+ *	uint64_t, but do not try this unless you know what you are doing.
+ */
 typedef size_t	mdb_size_t;
 # define MDB_SIZE_MAX	SIZE_MAX	/**< max #mdb_size_t */
 /** #mdb_size_t printf formats, \b t = one of [diouxX] without quotes */
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 0d2889fd29..8e5b907015 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5414,7 +5414,7 @@ mdb_env_close(MDB_env *env)
 	free(env);
 }
 
-/** Compare two items pointing at aligned mdb_size_t's */
+/** Compare two items pointing at aligned #mdb_size_t's */
 static int
 mdb_cmp_long(const MDB_val *a, const MDB_val *b)
 {
@@ -5425,7 +5425,7 @@ mdb_cmp_long(const MDB_val *a, const MDB_val *b)
 /** Compare two items pointing at aligned unsigned int's.
  *
  *	This is also set as #MDB_INTEGERDUP|#MDB_DUPFIXED's #MDB_dbx.%md_dcmp,
- *	but #mdb_cmp_clong() is called instead if the data type is mdb_size_t.
+ *	but #mdb_cmp_clong() is called instead if the data type is #mdb_size_t.
  */
 static int
 mdb_cmp_int(const MDB_val *a, const MDB_val *b)

From 8670805b4fb379d70dea0b94dd437238e8d694f6 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 2 Aug 2016 22:16:09 +0200
Subject: [PATCH 250/504] Doc fixes: VALID_FLAGS, mm_last_pg, mt_loose_count

---
 libraries/liblmdb/mdb.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 8e5b907015..00e5ffe04b 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1074,9 +1074,9 @@ typedef struct MDB_db {
 	pgno_t		md_root;		/**< the root page of this tree */
 } MDB_db;
 
-	/** mdb_dbi_open flags */
 #define MDB_VALID	0x8000		/**< DB handle is valid, for me_dbflags */
 #define PERSISTENT_FLAGS	(0xffff & ~(MDB_VALID))
+	/** #mdb_dbi_open() flags */
 #define VALID_FLAGS	(MDB_REVERSEKEY|MDB_DUPSORT|MDB_INTEGERKEY|MDB_DUPFIXED|\
 	MDB_INTEGERDUP|MDB_REVERSEDUP|MDB_CREATE)
 
@@ -1115,7 +1115,10 @@ typedef struct MDB_meta {
 #define	mm_psize	mm_dbs[FREE_DBI].md_pad
 	/** Any persistent environment flags. @ref mdb_env */
 #define	mm_flags	mm_dbs[FREE_DBI].md_flags
-	pgno_t		mm_last_pg;			/**< last used page in file */
+	/** Last used page in the datafile.
+	 *	Actually the file may be shorter if the freeDB lists the final pages.
+	 */
+	pgno_t		mm_last_pg;
 	volatile txnid_t	mm_txnid;	/**< txnid that committed this page */
 } MDB_meta;
 
@@ -1168,7 +1171,7 @@ struct MDB_txn {
 	 *	in this transaction, linked through #NEXT_LOOSE_PAGE(page).
 	 */
 	MDB_page	*mt_loose_pgs;
-	/* #Number of loose pages (#mt_loose_pgs) */
+	/** Number of loose pages (#mt_loose_pgs) */
 	int			mt_loose_count;
 	/** The sorted list of dirty pages we temporarily wrote to disk
 	 *	because the dirty list was full. page numbers in here are

From 77d522d15158d9fe54c2509b8998227c97cbdcdb Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 2 Aug 2016 22:17:54 +0200
Subject: [PATCH 251/504] Comment MDB_page

---
 libraries/liblmdb/mdb.c | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 00e5ffe04b..5c2aeb11ea 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -868,9 +868,23 @@ typedef struct MDB_txninfo {
 	  + (((MDB_PIDLOCK) != 0) << 16)))
 /** @} */
 
-/** Common header for all page types.
- * Overflow records occupy a number of contiguous pages with no
- * headers on any page after the first.
+/** Common header for all page types. The page type depends on #mp_flags.
+ *
+ * #P_BRANCH and #P_LEAF pages have unsorted '#MDB_node's at the end, with
+ * sorted #mp_ptrs[] entries referring to them. Exception: #P_LEAF2 pages
+ * omit mp_ptrs and pack sorted #MDB_DUPFIXED values after the page header.
+ *
+ * #P_OVERFLOW records occupy one or more contiguous pages where only the
+ * first has a page header. They hold the real data of #F_BIGDATA nodes.
+ *
+ * #P_SUBP sub-pages are small leaf "pages" with duplicate data.
+ * A node with flag #F_DUPDATA but not #F_SUBDATA contains a sub-page.
+ * (Duplicate data can also go in sub-databases, which use normal pages.)
+ *
+ * #P_META pages contain #MDB_meta, the start point of an LMDB snapshot.
+ *
+ * Each non-metapage up to #MDB_meta.%mm_last_pg is reachable exactly once
+ * in the snapshot: Either used by a database or listed in a freeDB record.
  */
 typedef struct MDB_page {
 #define	mp_pgno	mp_p.p_pgno
@@ -879,7 +893,7 @@ typedef struct MDB_page {
 		pgno_t		p_pgno;	/**< page number */
 		struct MDB_page *p_next; /**< for in-memory list of freed pages */
 	} mp_p;
-	uint16_t	mp_pad;
+	uint16_t	mp_pad;			/**< key size if this is a LEAF2 page */
 /**	@defgroup mdb_page	Page Flags
  *	@ingroup internal
  *	Flags for the page headers.
@@ -946,7 +960,9 @@ typedef struct MDB_page {
 	/** The number of overflow pages needed to store the given size. */
 #define OVPAGES(size, psize)	((PAGEHDRSZ-1 + (size)) / (psize) + 1)
 
-	/** Link in #MDB_txn.%mt_loose_pgs list */
+	/** Link in #MDB_txn.%mt_loose_pgs list.
+	 *  Kept outside the page header, which is needed when reusing the page.
+	 */
 #define NEXT_LOOSE_PAGE(p)		(*(MDB_page **)((p) + 2))
 
 	/** Header for a single key/data pair within a page.

From 7e2290c52bb4e8543a84f4e7f6c9a4f4d8ebf5d2 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 11 Aug 2016 21:30:56 +0100
Subject: [PATCH 252/504] MDB_VL32 plug rpage leak

mdb_cursor_set wasn't unref'ing as intended.
---
 libraries/liblmdb/mdb.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 5c2aeb11ea..2af32e052b 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -6620,8 +6620,10 @@ mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data,
 	if (key->mv_size == 0)
 		return MDB_BAD_VALSIZE;
 
-	if (mc->mc_xcursor)
+	if (mc->mc_xcursor) {
+		MDB_CURSOR_UNREF(&mc->mc_xcursor->mx_cursor, 0);
 		mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF);
+	}
 
 	/* See if we're already on the right page */
 	if (mc->mc_flags & C_INITIALIZED) {
@@ -6753,8 +6755,6 @@ set1:
 		return MDB_SUCCESS;
 	}
 
-	if (mc->mc_xcursor)
-		MDB_CURSOR_UNREF(&mc->mc_xcursor->mx_cursor, 0);
 	if (F_ISSET(leaf->mn_flags, F_DUPDATA)) {
 		mdb_xcursor_init1(mc, leaf);
 	}
@@ -8303,6 +8303,9 @@ mdb_cursor_count(MDB_cursor *mc, mdb_size_t *countp)
 void
 mdb_cursor_close(MDB_cursor *mc)
 {
+	if (mc) {
+		MDB_CURSOR_UNREF(mc, 0);
+	}
 	if (mc && !mc->mc_backup) {
 		/* remove from txn, if tracked */
 		if ((mc->mc_flags & C_UNTRACK) && mc->mc_txn->mt_cursors) {

From c6510147cee88d182af431c14a61fad6fae18be3 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Fri, 19 Aug 2016 17:24:25 +0100
Subject: [PATCH 253/504] ITS#8481 make shared lib suffix overridable

---
 libraries/liblmdb/Makefile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile
index f3c93a2ff5..72d0984607 100644
--- a/libraries/liblmdb/Makefile
+++ b/libraries/liblmdb/Makefile
@@ -26,6 +26,7 @@ OPT = -O2 -g
 CFLAGS	= $(THREADS) $(OPT) $(W) $(XCFLAGS)
 LDLIBS	= # -lntdll # Windows needs ntdll
 SOLIBS	= # -lntdll
+SOEXT	= .so
 prefix	= /usr/local
 exec_prefix = $(prefix)
 bindir = $(exec_prefix)/bin
@@ -37,7 +38,7 @@ mandir = $(datarootdir)/man
 ########################################################################
 
 IHDRS	= lmdb.h
-ILIBS	= liblmdb.a liblmdb.so
+ILIBS	= liblmdb.a liblmdb$(SOEXT)
 IPROGS	= mdb_stat mdb_copy mdb_dump mdb_load
 IDOCS	= mdb_stat.1 mdb_copy.1 mdb_dump.1 mdb_load.1
 PROGS	= $(IPROGS) mtest mtest2 mtest3 mtest4 mtest5
@@ -63,7 +64,7 @@ test:	all
 liblmdb.a:	mdb.o midl.o
 	$(AR) rs $@ mdb.o midl.o
 
-liblmdb.so:	mdb.lo midl.lo
+liblmdb$(SOEXT):	mdb.lo midl.lo
 #	$(CC) $(LDFLAGS) -pthread -shared -Wl,-Bsymbolic -o $@ mdb.o midl.o $(SOLIBS)
 	$(CC) $(LDFLAGS) -pthread -shared -o $@ mdb.lo midl.lo $(SOLIBS)
 

From 36d374ba9eb75fe9812cf69a29bae69e94f559f9 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sun, 21 Aug 2016 23:20:22 +0200
Subject: [PATCH 254/504] Fix mdb_page_list() message

---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 2af32e052b..3e16b87770 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1739,7 +1739,7 @@ mdb_page_list(MDB_page *mp)
 			pgno, ((MDB_meta *)METADATA(mp))->mm_txnid);
 		return;
 	default:
-		fprintf(stderr, "Bad page %"Yu" flags 0x%u\n", pgno, mp->mp_flags);
+		fprintf(stderr, "Bad page %"Yu" flags 0x%X\n", pgno, mp->mp_flags);
 		return;
 	}
 

From da4443a9b3f6b3ecf8594a706e0403eaf301727f Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sun, 21 Aug 2016 23:22:18 +0200
Subject: [PATCH 255/504] Clean up MDB_USE_ROBUST #defines

Make explicit and default nonzero equivalent. Parenthesize.
---
 libraries/liblmdb/mdb.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 3e16b87770..32f0a5a881 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -303,6 +303,10 @@ union semun {
 #  define MDB_USE_ROBUST	0
 # else
 #  define MDB_USE_ROBUST	1
+# endif
+#endif /* !MDB_USE_ROBUST */
+
+#if defined(MDB_USE_POSIX_MUTEX) && (MDB_USE_ROBUST)
 /* glibc < 2.12 only provided _np API */
 #  if (defined(__GLIBC__) && GLIBC_VER < 0x02000c) || \
 	(defined(PTHREAD_MUTEX_ROBUST_NP) && !defined(PTHREAD_MUTEX_ROBUST))
@@ -310,10 +314,9 @@ union semun {
 #   define pthread_mutexattr_setrobust(attr, flag)	pthread_mutexattr_setrobust_np(attr, flag)
 #   define pthread_mutex_consistent(mutex)	pthread_mutex_consistent_np(mutex)
 #  endif
-# endif
-#endif /* MDB_USE_ROBUST */
+#endif /* MDB_USE_POSIX_MUTEX && MDB_USE_ROBUST */
 
-#if defined(MDB_OWNERDEAD) && MDB_USE_ROBUST
+#if defined(MDB_OWNERDEAD) && (MDB_USE_ROBUST)
 #define MDB_ROBUST_SUPPORTED	1
 #endif
 

From 45a88275d2a410e683bae4ef44881e0f55fa3c4d Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 1 Sep 2016 00:41:35 +0100
Subject: [PATCH 256/504] ITS#8489 reset cursor EOF flag in cursor_set

It usually gets done anyway, but one of the fastpath shortcuts
bypassed this step.
---
 libraries/liblmdb/mdb.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 32f0a5a881..0110402c5b 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -6692,6 +6692,7 @@ mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data,
 						}
 					}
 					rc = 0;
+					mc->mc_flags &= ~C_EOF;
 					goto set2;
 				}
 			}

From 26e226b2a8cdba4208eabca394537900c4eda3b3 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 3 Sep 2016 09:41:30 +0200
Subject: [PATCH 257/504] Silence warning for initializer "mdb_copy my = {0}"

1st struct member was not a scalar.
---
 libraries/liblmdb/mdb.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 0110402c5b..b0518a5562 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -9628,12 +9628,12 @@ mdb_put(MDB_txn *txn, MDB_dbi dbi,
 
 	/** State needed for a double-buffering compacting copy. */
 typedef struct mdb_copy {
+	MDB_env *mc_env;
+	MDB_txn *mc_txn;
 	pthread_mutex_t mc_mutex;
 	pthread_cond_t mc_cond;	/**< Condition variable for #mc_new */
 	char *mc_wbuf[2];
 	char *mc_over[2];
-	MDB_env *mc_env;
-	MDB_txn *mc_txn;
 	int mc_wlen[2];
 	int mc_olen[2];
 	pgno_t mc_next_pgno;

From cdcf9da600a5ffce71b8dba212c33b3e72bdbb8c Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 6 Sep 2016 17:41:02 +0200
Subject: [PATCH 258/504] ITS#7682 F_NOCACHE: Allow error, skip any O_DIRECT

We can run without F_NOCACHE if it fails. And we do not know
what combining it with O_DIRECT means, if a system has both.
---
 libraries/liblmdb/mdb.c | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index b0518a5562..ecfd5cab66 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -10169,17 +10169,12 @@ mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags)
 	}
 
 	if (env->me_psize >= env->me_os_psize) {
-#ifdef O_DIRECT
+#ifdef F_NOCACHE	/* __APPLE__ */
+	(void) fcntl(newfd, F_NOCACHE, 1);
+#elif defined O_DIRECT
 	/* Set O_DIRECT if the file system supports it */
 	if ((rc = fcntl(newfd, F_GETFL)) != -1)
 		(void) fcntl(newfd, F_SETFL, rc | O_DIRECT);
-#endif
-#ifdef F_NOCACHE	/* __APPLE__ */
-	rc = fcntl(newfd, F_NOCACHE, 1);
-	if (rc) {
-		rc = ErrCode();
-		goto leave;
-	}
 #endif
 	}
 

From 3dd2d207d7e2cdd0fe040b919be56a1e23851aea Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 6 Sep 2016 17:48:31 +0200
Subject: [PATCH 259/504] Drop spurious Errcode() call

---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index ecfd5cab66..b8ecfd8779 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -4846,7 +4846,7 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 		if (rc == MDB_ERRCODE_ROFS && (env->me_flags & MDB_RDONLY)) {
 			return MDB_SUCCESS;
 		}
-		goto fail_errno;
+		goto fail;
 	}
 #if ! ((MDB_CLOEXEC) || defined(_WIN32))
 	/* Lose record locks when exec*() */

From d87ee20e0bf7fc751f75412082df307647165d56 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 6 Sep 2016 17:56:13 +0200
Subject: [PATCH 260/504] Clean up strange fcntl result check

...and check !MDB_CLOEXEC in an 'if' rather than '#if'
to match its non-zero usage.
---
 libraries/liblmdb/mdb.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index b8ecfd8779..087cb98bc1 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -4818,9 +4818,9 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 #ifdef O_CLOEXEC	/* Linux: Open file and set FD_CLOEXEC atomically */
 #	define MDB_CLOEXEC		O_CLOEXEC
 #else
-	int fdflags;
 #	define MDB_CLOEXEC		0
 #endif
+	int fdflags;
 #endif
 #ifdef MDB_USE_SYSV_SEM
 	int semid;
@@ -4848,10 +4848,10 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 		}
 		goto fail;
 	}
-#if ! ((MDB_CLOEXEC) || defined(_WIN32))
+#ifndef _WIN32
 	/* Lose record locks when exec*() */
-	if ((fdflags = fcntl(env->me_lfd, F_GETFD) | FD_CLOEXEC) >= 0)
-			fcntl(env->me_lfd, F_SETFD, fdflags);
+	if (!(MDB_CLOEXEC) && (fdflags = fcntl(env->me_lfd, F_GETFD)) != -1)
+			fcntl(env->me_lfd, F_SETFD, fdflags | FD_CLOEXEC);
 #endif
 
 	if (!(env->me_flags & MDB_NOTLS)) {

From 67fb3c746a43bf2d2ce4834ede5fba940c9286a5 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 6 Sep 2016 18:12:01 +0200
Subject: [PATCH 261/504] ITS#7992 Tighter utf8_to_utf16(), fix errcodes

The 0xFFFD check seems due to misleading MultiByteToWideChar() doc.
Bad UTF-8 gives 0xFFFD in the output string, not the return value.
---
 libraries/liblmdb/mdb.c | 40 +++++++++++++++++++++++-----------------
 1 file changed, 23 insertions(+), 17 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 087cb98bc1..b0da3856f9 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -10886,25 +10886,31 @@ mdb_mutex_failed(MDB_env *env, mdb_mutexref_t mutex, int rc)
 	return rc;
 }
 #endif	/* MDB_ROBUST_SUPPORTED */
-/** @} */
 
 #if defined(_WIN32)
-static int utf8_to_utf16(const char *src, int srcsize, wchar_t **dst, int *dstsize)
+static int ESECT
+utf8_to_utf16(const char *src, int srcsize, wchar_t **dst, int *dstsize)
 {
-	int need;
-	wchar_t *result;
-	need = MultiByteToWideChar(CP_UTF8, 0, src, srcsize, NULL, 0);
-	if (need == 0xFFFD)
-		return EILSEQ;
-	if (need == 0)
-		return EINVAL;
-	result = malloc(sizeof(wchar_t) * need);
-	if (!result)
-		return ENOMEM;
-	MultiByteToWideChar(CP_UTF8, 0, src, srcsize, result, need);
-	if (dstsize)
-		*dstsize = need;
-	*dst = result;
-	return 0;
+	int rc, need = 0;
+	wchar_t *result = NULL;
+	for (;;) {					/* malloc result, then fill it in */
+		need = MultiByteToWideChar(CP_UTF8, 0, src, srcsize, result, need);
+		if (!need) {
+			rc = ErrCode();
+			free(result);
+			return rc;
+		}
+		if (!result) {
+			result = malloc(sizeof(wchar_t) * need);
+			if (!result)
+				return ENOMEM;
+			continue;
+		}
+		if (dstsize)
+			*dstsize = need;
+		*dst = result;
+		return MDB_SUCCESS;
+	}
 }
 #endif /* defined(_WIN32) */
+/** @} */

From cdc3f9cc2d1d440920352b2f4524e28a93445e3a Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 17 Sep 2016 21:31:04 +0200
Subject: [PATCH 262/504] Factor filename handling out to mdb_fname_*()

No change in functionality, except needs less mallocing.
---
 libraries/liblmdb/mdb.c | 199 +++++++++++++++++++++++-----------------
 1 file changed, 113 insertions(+), 86 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index b0da3856f9..763972bdf8 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -41,6 +41,7 @@
 #ifdef _WIN32
 #include <malloc.h>
 #include <windows.h>
+#include <wchar.h>				/* get wcscpy() */
 
 /* We use native NT APIs to setup the memory map, so that we can
  * let the DB file grow incrementally instead of always preallocating
@@ -1554,7 +1555,8 @@ static SECURITY_DESCRIPTOR mdb_null_sd;
 static SECURITY_ATTRIBUTES mdb_all_sa;
 static int mdb_sec_inited;
 
-static int utf8_to_utf16(const char *src, int srcsize, wchar_t **dst, int *dstsize);
+struct MDB_name;
+static int utf8_to_utf16(const char *src, struct MDB_name *dst, int xtra);
 #endif
 
 /** Return the library version info. */
@@ -4386,6 +4388,79 @@ mdb_fsize(HANDLE fd, mdb_size_t *size)
 	return MDB_SUCCESS;
 }
 
+
+#ifdef _WIN32
+typedef wchar_t	mdb_nchar_t;
+# define MDB_NAME(str)	L##str
+# define mdb_name_cpy	wcscpy
+#else
+/** Character type for file names: char on Unix, wchar_t on Windows */
+typedef char	mdb_nchar_t;
+# define MDB_NAME(str)	str		/**< #mdb_nchar_t[] string literal */
+# define mdb_name_cpy	strcpy	/**< Copy name (#mdb_nchar_t string) */
+#endif
+
+/** Filename - string of #mdb_nchar_t[] */
+typedef struct MDB_name {
+	int mn_len;					/**< Length  */
+	int mn_alloced;				/**< True if #mn_val was malloced */
+	mdb_nchar_t	*mn_val;		/**< Contents */
+} MDB_name;
+
+/** Filename suffixes [datafile,lockfile][without,with MDB_NOSUBDIR] */
+static const mdb_nchar_t *const mdb_suffixes[2][2] = {
+	{ MDB_NAME("/data.mdb"), MDB_NAME("")      },
+	{ MDB_NAME("/lock.mdb"), MDB_NAME("-lock") }
+};
+
+#define MDB_SUFFLEN 9	/**< Max string length in #mdb_suffixes[] */
+
+/** Set up filename + scratch area for filename suffix, for opening files.
+ * It should be freed with #mdb_fname_destroy().
+ * On Windows, paths are converted from char *UTF-8 to wchar_t *UTF-16.
+ *
+ * @param[in] path Pathname for #mdb_env_open().
+ * @param[in] envflags Whether a subdir and/or lockfile will be used.
+ * @param[out] fname Resulting filename, with room for a suffix if necessary.
+ */
+static int ESECT
+mdb_fname_init(const char *path, unsigned envflags, MDB_name *fname)
+{
+	int no_suffix = F_ISSET(envflags, MDB_NOSUBDIR|MDB_NOLOCK);
+	fname->mn_alloced = 0;
+#ifdef _WIN32
+	return utf8_to_utf16(path, fname, no_suffix ? 0 : MDB_SUFFLEN);
+#else
+	fname->mn_len = strlen(path);
+	if (no_suffix)
+		fname->mn_val = (char *) path;
+	else if ((fname->mn_val = malloc(fname->mn_len + MDB_SUFFLEN+1)) != NULL) {
+		fname->mn_alloced = 1;
+		strcpy(fname->mn_val, path);
+	}
+	else
+		return ENOMEM;
+	return MDB_SUCCESS;
+#endif
+}
+
+/** Destroy \b fname from #mdb_fname_init() */
+#define mdb_fname_destroy(fname) \
+	do { if ((fname).mn_alloced) free((fname).mn_val); } while (0)
+
+enum mdb_fopen_type {
+	MDB_O_DATA, MDB_O_LOCKS
+};
+
+static void ESECT
+mdb_fname_suffix_set(MDB_env *env, MDB_name *fname, enum mdb_fopen_type which)
+{
+	if (fname->mn_alloced)		/* modifiable copy */
+		mdb_name_cpy(fname->mn_val + fname->mn_len,
+			mdb_suffixes[which==MDB_O_LOCKS][F_ISSET(env->me_flags, MDB_NOSUBDIR)]);
+}
+
+
 #ifdef BROKEN_FDATASYNC
 #include <sys/utsname.h>
 #include <sys/vfs.h>
@@ -4803,13 +4878,13 @@ mdb_hash_enc(MDB_val *val, char *encbuf)
 
 /** Open and/or initialize the lock region for the environment.
  * @param[in] env The LMDB environment.
- * @param[in] lpath The pathname of the file used for the lock region.
+ * @param[in] fname Filename + scratch area, from #mdb_fname_init().
  * @param[in] mode The Unix permissions for the file, if we create it.
  * @param[in,out] excl In -1, out lock type: -1 none, 0 shared, 1 exclusive
  * @return 0 on success, non-zero on failure.
  */
 static int ESECT
-mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
+mdb_env_setup_locks(MDB_env *env, MDB_name *fname, int mode, int *excl)
 {
 #ifdef _WIN32
 #	define MDB_ERRCODE_ROFS	ERROR_WRITE_PROTECT
@@ -4829,17 +4904,13 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 	int rc;
 	off_t size, rsize;
 
+	mdb_fname_suffix_set(env, fname, MDB_O_LOCKS);
 #ifdef _WIN32
-	wchar_t *wlpath;
-	rc = utf8_to_utf16(lpath, -1, &wlpath, NULL);
-	if (rc)
-		return rc;
-	env->me_lfd = CreateFileW(wlpath, GENERIC_READ|GENERIC_WRITE,
+	env->me_lfd = CreateFileW(fname->mn_val, GENERIC_READ|GENERIC_WRITE,
 		FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, OPEN_ALWAYS,
 		FILE_ATTRIBUTE_NORMAL, NULL);
-	free(wlpath);
 #else
-	env->me_lfd = open(lpath, O_RDWR|O_CREAT|MDB_CLOEXEC, mode);
+	env->me_lfd = open(fname->mn_val, O_RDWR|O_CREAT|MDB_CLOEXEC, mode);
 #endif
 	if (env->me_lfd == INVALID_HANDLE_VALUE) {
 		rc = ErrCode();
@@ -4979,7 +5050,7 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 		if (env->me_wmutex == SEM_FAILED) goto fail_errno;
 #elif defined(MDB_USE_SYSV_SEM)
 		unsigned short vals[2] = {1, 1};
-		key_t key = ftok(lpath, 'M');
+		key_t key = ftok(fname->mn_val, 'M'); /* fname is lockfile path now */
 		if (key == -1)
 			goto fail_errno;
 		semid = semget(key, 2, (mode & 0777) | IPC_CREAT);
@@ -5083,12 +5154,6 @@ fail:
 	return rc;
 }
 
-	/** The name of the lock file in the DB environment */
-#define LOCKNAME	"/lock.mdb"
-	/** The name of the data file in the DB environment */
-#define DATANAME	"/data.mdb"
-	/** The suffix of the lock file when no subdir is used */
-#define LOCKSUFF	"-lock"
 	/** Only a subset of the @ref mdb_env flags can be changed
 	 *	at runtime. Changing other flags requires closing the
 	 *	environment and re-opening it with the new flags.
@@ -5105,10 +5170,7 @@ int ESECT
 mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode)
 {
 	int		oflags, rc, len, excl = -1;
-	char *lpath, *dpath;
-#ifdef _WIN32
-	wchar_t *wpath;
-#endif
+	MDB_name fname;
 
 	if (env->me_fd!=INVALID_HANDLE_VALUE || (flags & ~(CHANGEABLE|CHANGELESS)))
 		return EINVAL;
@@ -5123,28 +5185,12 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 		return EINVAL;
 	}
 #endif
-
-	len = strlen(path);
-	if (flags & MDB_NOSUBDIR) {
-		rc = len + sizeof(LOCKSUFF) + len + 1;
-	} else {
-		rc = len + sizeof(LOCKNAME) + len + sizeof(DATANAME);
-	}
-	lpath = malloc(rc);
-	if (!lpath)
-		return ENOMEM;
-	if (flags & MDB_NOSUBDIR) {
-		dpath = lpath + len + sizeof(LOCKSUFF);
-		sprintf(lpath, "%s" LOCKSUFF, path);
-		strcpy(dpath, path);
-	} else {
-		dpath = lpath + len + sizeof(LOCKNAME);
-		sprintf(lpath, "%s" LOCKNAME, path);
-		sprintf(dpath, "%s" DATANAME, path);
-	}
-
-	rc = MDB_SUCCESS;
 	flags |= env->me_flags;
+
+	rc = mdb_fname_init(path, flags, &fname);
+	if (rc)
+		return rc;
+
 	if (flags & MDB_RDONLY) {
 		/* silently ignore WRITEMAP when we're only getting read access */
 		flags &= ~MDB_WRITEMAP;
@@ -5180,11 +5226,12 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 
 	/* For RDONLY, get lockfile after we know datafile exists */
 	if (!(flags & (MDB_RDONLY|MDB_NOLOCK))) {
-		rc = mdb_env_setup_locks(env, lpath, mode, &excl);
+		rc = mdb_env_setup_locks(env, &fname, mode, &excl);
 		if (rc)
 			goto leave;
 	}
 
+	mdb_fname_suffix_set(env, &fname, MDB_O_DATA);
 #ifdef _WIN32
 	if (F_ISSET(flags, MDB_RDONLY)) {
 		oflags = GENERIC_READ;
@@ -5194,19 +5241,15 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 		len = OPEN_ALWAYS;
 	}
 	mode = FILE_ATTRIBUTE_NORMAL;
-	rc = utf8_to_utf16(dpath, -1, &wpath, NULL);
-	if (rc)
-		goto leave;
-	env->me_fd = CreateFileW(wpath, oflags, FILE_SHARE_READ|FILE_SHARE_WRITE,
+	env->me_fd = CreateFileW(fname.mn_val, oflags, FILE_SHARE_READ|FILE_SHARE_WRITE,
 		NULL, len, mode, NULL);
-	free(wpath);
 #else
 	if (F_ISSET(flags, MDB_RDONLY))
 		oflags = O_RDONLY;
 	else
 		oflags = O_RDWR | O_CREAT;
 
-	env->me_fd = open(dpath, oflags, mode);
+	env->me_fd = open(fname.mn_val, oflags, mode);
 #endif
 	if (env->me_fd == INVALID_HANDLE_VALUE) {
 		rc = ErrCode();
@@ -5214,7 +5257,7 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 	}
 
 	if ((flags & (MDB_RDONLY|MDB_NOLOCK)) == MDB_RDONLY) {
-		rc = mdb_env_setup_locks(env, lpath, mode, &excl);
+		rc = mdb_env_setup_locks(env, &fname, mode, &excl);
 		if (rc)
 			goto leave;
 	}
@@ -5226,18 +5269,15 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 			/* Synchronous fd for meta writes. Needed even with
 			 * MDB_NOSYNC/MDB_NOMETASYNC, in case these get reset.
 			 */
+			mdb_fname_suffix_set(env, &fname, MDB_O_DATA);
 #ifdef _WIN32
 			len = OPEN_EXISTING;
-			rc = utf8_to_utf16(dpath, -1, &wpath, NULL);
-			if (rc)
-				goto leave;
-			env->me_mfd = CreateFileW(wpath, oflags,
+			env->me_mfd = CreateFileW(fname.mn_val, oflags,
 				FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, len,
 				mode | FILE_FLAG_WRITE_THROUGH, NULL);
-			free(wpath);
 #else
 			oflags &= ~O_CREAT;
-			env->me_mfd = open(dpath, oflags | MDB_DSYNC, mode);
+			env->me_mfd = open(fname.mn_val, oflags | MDB_DSYNC, mode);
 #endif
 			if (env->me_mfd == INVALID_HANDLE_VALUE) {
 				rc = ErrCode();
@@ -5285,7 +5325,7 @@ leave:
 	if (rc) {
 		mdb_env_close0(env, excl);
 	}
-	free(lpath);
+	mdb_fname_destroy(fname);
 	return rc;
 }
 
@@ -10131,37 +10171,24 @@ mdb_env_copyfd(MDB_env *env, HANDLE fd)
 int ESECT
 mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags)
 {
-	int rc, len;
-	char *lpath;
+	int rc;
+	MDB_name fname;
 	HANDLE newfd = INVALID_HANDLE_VALUE;
-#ifdef _WIN32
-	wchar_t *wpath;
-#endif
 
-	if (env->me_flags & MDB_NOSUBDIR) {
-		lpath = (char *)path;
-	} else {
-		len = strlen(path);
-		len += sizeof(DATANAME);
-		lpath = malloc(len);
-		if (!lpath)
-			return ENOMEM;
-		sprintf(lpath, "%s" DATANAME, path);
-	}
+	rc = mdb_fname_init(path, env->me_flags | MDB_NOLOCK, &fname);
+	if (rc)
+		return rc;
+	mdb_fname_suffix_set(env, &fname, MDB_O_DATA);
 
 	/* The destination path must exist, but the destination file must not.
 	 * We don't want the OS to cache the writes, since the source data is
 	 * already in the OS cache.
 	 */
 #ifdef _WIN32
-	rc = utf8_to_utf16(lpath, -1, &wpath, NULL);
-	if (rc)
-		goto leave;
-	newfd = CreateFileW(wpath, GENERIC_WRITE, 0, NULL, CREATE_NEW,
+	newfd = CreateFileW(fname.mn_val, GENERIC_WRITE, 0, NULL, CREATE_NEW,
 				FILE_FLAG_NO_BUFFERING|FILE_FLAG_WRITE_THROUGH, NULL);
-	free(wpath);
 #else
-	newfd = open(lpath, O_WRONLY|O_CREAT|O_EXCL, 0666);
+	newfd = open(fname.mn_val, O_WRONLY|O_CREAT|O_EXCL, 0666);
 #endif
 	if (newfd == INVALID_HANDLE_VALUE) {
 		rc = ErrCode();
@@ -10181,8 +10208,7 @@ mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags)
 	rc = mdb_env_copyfd2(env, newfd, flags);
 
 leave:
-	if (!(env->me_flags & MDB_NOSUBDIR))
-		free(lpath);
+	mdb_fname_destroy(fname);
 	if (newfd != INVALID_HANDLE_VALUE)
 		if (close(newfd) < 0 && rc == MDB_SUCCESS)
 			rc = ErrCode();
@@ -10888,27 +10914,28 @@ mdb_mutex_failed(MDB_env *env, mdb_mutexref_t mutex, int rc)
 #endif	/* MDB_ROBUST_SUPPORTED */
 
 #if defined(_WIN32)
+/** Convert \b src to new wchar_t[] string with room for \b xtra extra chars */
 static int ESECT
-utf8_to_utf16(const char *src, int srcsize, wchar_t **dst, int *dstsize)
+utf8_to_utf16(const char *src, MDB_name *dst, int xtra)
 {
 	int rc, need = 0;
 	wchar_t *result = NULL;
 	for (;;) {					/* malloc result, then fill it in */
-		need = MultiByteToWideChar(CP_UTF8, 0, src, srcsize, result, need);
+		need = MultiByteToWideChar(CP_UTF8, 0, src, -1, result, need);
 		if (!need) {
 			rc = ErrCode();
 			free(result);
 			return rc;
 		}
 		if (!result) {
-			result = malloc(sizeof(wchar_t) * need);
+			result = malloc(sizeof(wchar_t) * (need + xtra));
 			if (!result)
 				return ENOMEM;
 			continue;
 		}
-		if (dstsize)
-			*dstsize = need;
-		*dst = result;
+		dst->mn_alloced = 1;
+		dst->mn_len = need - 1;
+		dst->mn_val = result;
 		return MDB_SUCCESS;
 	}
 }

From 15666878afb57439abfaf5c5a1f3a338f4e23104 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 27 Sep 2016 07:03:34 +0200
Subject: [PATCH 263/504] Move opening files to mdb_fopen()

No change in functionality.
---
 libraries/liblmdb/mdb.c | 220 +++++++++++++++++++++++-----------------
 1 file changed, 125 insertions(+), 95 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 763972bdf8..0a7fcc02e8 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -4448,16 +4448,122 @@ mdb_fname_init(const char *path, unsigned envflags, MDB_name *fname)
 #define mdb_fname_destroy(fname) \
 	do { if ((fname).mn_alloced) free((fname).mn_val); } while (0)
 
+#ifdef O_CLOEXEC /* POSIX.1-2008: Set FD_CLOEXEC atomically at open() */
+# define MDB_CLOEXEC		O_CLOEXEC
+#else
+# define MDB_CLOEXEC		0
+#endif
+
+/** File type, access mode etc. for #mdb_fopen() */
 enum mdb_fopen_type {
-	MDB_O_DATA, MDB_O_LOCKS
+#ifdef _WIN32
+	MDB_O_RDONLY, MDB_O_RDWR, MDB_O_META, MDB_O_COPY, MDB_O_LOCKS
+#else
+	/* A comment in mdb_fopen() explains some O_* flag choices. */
+	MDB_O_RDONLY= O_RDONLY,                            /**< for RDONLY me_fd */
+	MDB_O_RDWR  = O_RDWR  |O_CREAT,                    /**< for me_fd */
+	MDB_O_META  = O_RDWR  |MDB_DSYNC,                  /**< for me_mfd */
+	MDB_O_COPY  = O_WRONLY|O_CREAT|O_EXCL,             /**< for #mdb_env_copy() */
+	/** Bitmask for open() flags in enum #mdb_fopen_type.  The other bits
+	 * distinguish otherwise-equal MDB_O_* constants from each other.
+	 */
+	MDB_O_MASK  = MDB_O_RDWR|MDB_CLOEXEC | MDB_O_RDONLY|MDB_O_META|MDB_O_COPY,
+	MDB_O_LOCKS = MDB_O_RDWR|MDB_CLOEXEC | ((MDB_O_MASK+1) & ~MDB_O_MASK) /**< for me_lfd */
+#endif
 };
 
-static void ESECT
-mdb_fname_suffix_set(MDB_env *env, MDB_name *fname, enum mdb_fopen_type which)
+/** Open an LMDB file.
+ * @param[in] env	The LMDB environment.
+ * @param[in,out] fname	Path from from #mdb_fname_init().  A suffix is
+ * appended if necessary to create the filename, without changing mn_len.
+ * @param[in] which	Determines file type, access mode, etc.
+ * @param[in] mode	The Unix permissions for the file, if we create it.
+ * @param[out] res	Resulting file handle.
+ * @return 0 on success, non-zero on failure.
+ */
+static int ESECT
+mdb_fopen(const MDB_env *env, MDB_name *fname,
+	enum mdb_fopen_type which, mdb_mode_t mode,
+	HANDLE *res)
 {
+	int rc = MDB_SUCCESS;
+	HANDLE fd;
+#ifdef _WIN32
+	DWORD acc, share, disp, attrs;
+#else
+	int flags;
+#endif
+
 	if (fname->mn_alloced)		/* modifiable copy */
 		mdb_name_cpy(fname->mn_val + fname->mn_len,
 			mdb_suffixes[which==MDB_O_LOCKS][F_ISSET(env->me_flags, MDB_NOSUBDIR)]);
+
+	/* The directory must already exist.  Usually the file need not.
+	 * MDB_O_META requires the file because we already created it using
+	 * MDB_O_RDWR.  MDB_O_COPY must not overwrite an existing file.
+	 *
+	 * With MDB_O_COPY we do not want the OS to cache the writes, since
+	 * the source data is already in the OS cache.
+	 *
+	 * The lockfile needs FD_CLOEXEC (close file descriptor on exec*())
+	 * to avoid the flock() issues noted under Caveats in lmdb.h.
+	 */
+
+#ifdef _WIN32
+	acc = GENERIC_READ|GENERIC_WRITE;
+	share = FILE_SHARE_READ|FILE_SHARE_WRITE;
+	disp = OPEN_ALWAYS;
+	attrs = FILE_ATTRIBUTE_NORMAL;
+	switch (which) {
+	case MDB_O_RDONLY:			/* read-only datafile */
+		acc = GENERIC_READ;
+		disp = OPEN_EXISTING;
+		break;
+	case MDB_O_META:			/* for writing metapages */
+		disp = OPEN_EXISTING;
+		attrs = FILE_ATTRIBUTE_NORMAL|FILE_FLAG_WRITE_THROUGH;
+		break;
+	case MDB_O_COPY:			/* mdb_env_copy() & co */
+		acc = GENERIC_WRITE;
+		share = 0;
+		disp = CREATE_NEW;
+		attrs = FILE_FLAG_NO_BUFFERING|FILE_FLAG_WRITE_THROUGH;
+		break;
+	default: break;	/* silence gcc -Wswitch (not all enum values handled) */
+	}
+	fd = CreateFileW(fname->mn_val, acc, share, NULL, disp, attrs, NULL);
+#else
+	fd = open(fname->mn_val, which & MDB_O_MASK, mode);
+#endif
+
+	if (fd == INVALID_HANDLE_VALUE)
+		rc = ErrCode();
+#ifndef _WIN32
+	else {
+		if (which == MDB_O_LOCKS) {
+			/* Set CLOEXEC if we could not pass it to open() */
+			if (!MDB_CLOEXEC && (flags = fcntl(fd, F_GETFD)) != -1)
+				(void) fcntl(fd, F_SETFD, flags | FD_CLOEXEC);
+		}
+		if (which == MDB_O_COPY && env->me_psize >= env->me_os_psize) {
+			/* This may require buffer alignment.  There is no portable
+			 * way to ask how much, so we require OS pagesize alignment.
+			 */
+# ifdef F_NOCACHE	/* __APPLE__ */
+			(void) fcntl(fd, F_NOCACHE, 1);
+# elif defined O_DIRECT
+			/* open(...O_DIRECT...) would break on filesystems without
+			 * O_DIRECT support (ITS#7682). Try to set it here instead.
+			 */
+			if ((flags = fcntl(fd, F_GETFL)) != -1)
+				(void) fcntl(fd, F_SETFL, flags | O_DIRECT);
+# endif
+		}
+	}
+#endif	/* !_WIN32 */
+
+	*res = fd;
+	return rc;
 }
 
 
@@ -4890,12 +4996,6 @@ mdb_env_setup_locks(MDB_env *env, MDB_name *fname, int mode, int *excl)
 #	define MDB_ERRCODE_ROFS	ERROR_WRITE_PROTECT
 #else
 #	define MDB_ERRCODE_ROFS	EROFS
-#ifdef O_CLOEXEC	/* Linux: Open file and set FD_CLOEXEC atomically */
-#	define MDB_CLOEXEC		O_CLOEXEC
-#else
-#	define MDB_CLOEXEC		0
-#endif
-	int fdflags;
 #endif
 #ifdef MDB_USE_SYSV_SEM
 	int semid;
@@ -4904,26 +5004,14 @@ mdb_env_setup_locks(MDB_env *env, MDB_name *fname, int mode, int *excl)
 	int rc;
 	off_t size, rsize;
 
-	mdb_fname_suffix_set(env, fname, MDB_O_LOCKS);
-#ifdef _WIN32
-	env->me_lfd = CreateFileW(fname->mn_val, GENERIC_READ|GENERIC_WRITE,
-		FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, OPEN_ALWAYS,
-		FILE_ATTRIBUTE_NORMAL, NULL);
-#else
-	env->me_lfd = open(fname->mn_val, O_RDWR|O_CREAT|MDB_CLOEXEC, mode);
-#endif
-	if (env->me_lfd == INVALID_HANDLE_VALUE) {
-		rc = ErrCode();
+	rc = mdb_fopen(env, fname, MDB_O_LOCKS, mode, &env->me_lfd);
+	if (rc) {
+		/* Omit lockfile if read-only env on read-only filesystem */
 		if (rc == MDB_ERRCODE_ROFS && (env->me_flags & MDB_RDONLY)) {
 			return MDB_SUCCESS;
 		}
 		goto fail;
 	}
-#ifndef _WIN32
-	/* Lose record locks when exec*() */
-	if (!(MDB_CLOEXEC) && (fdflags = fcntl(env->me_lfd, F_GETFD)) != -1)
-			fcntl(env->me_lfd, F_SETFD, fdflags | FD_CLOEXEC);
-#endif
 
 	if (!(env->me_flags & MDB_NOTLS)) {
 		rc = pthread_key_create(&env->me_txkey, mdb_env_reader_dest);
@@ -5169,7 +5257,7 @@ fail:
 int ESECT
 mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode)
 {
-	int		oflags, rc, len, excl = -1;
+	int rc, excl = -1;
 	MDB_name fname;
 
 	if (env->me_fd!=INVALID_HANDLE_VALUE || (flags & ~(CHANGEABLE|CHANGELESS)))
@@ -5231,30 +5319,11 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 			goto leave;
 	}
 
-	mdb_fname_suffix_set(env, &fname, MDB_O_DATA);
-#ifdef _WIN32
-	if (F_ISSET(flags, MDB_RDONLY)) {
-		oflags = GENERIC_READ;
-		len = OPEN_EXISTING;
-	} else {
-		oflags = GENERIC_READ|GENERIC_WRITE;
-		len = OPEN_ALWAYS;
-	}
-	mode = FILE_ATTRIBUTE_NORMAL;
-	env->me_fd = CreateFileW(fname.mn_val, oflags, FILE_SHARE_READ|FILE_SHARE_WRITE,
-		NULL, len, mode, NULL);
-#else
-	if (F_ISSET(flags, MDB_RDONLY))
-		oflags = O_RDONLY;
-	else
-		oflags = O_RDWR | O_CREAT;
-
-	env->me_fd = open(fname.mn_val, oflags, mode);
-#endif
-	if (env->me_fd == INVALID_HANDLE_VALUE) {
-		rc = ErrCode();
+	rc = mdb_fopen(env, &fname,
+		(flags & MDB_RDONLY) ? MDB_O_RDONLY : MDB_O_RDWR,
+		mode, &env->me_fd);
+	if (rc)
 		goto leave;
-	}
 
 	if ((flags & (MDB_RDONLY|MDB_NOLOCK)) == MDB_RDONLY) {
 		rc = mdb_env_setup_locks(env, &fname, mode, &excl);
@@ -5269,20 +5338,9 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 			/* Synchronous fd for meta writes. Needed even with
 			 * MDB_NOSYNC/MDB_NOMETASYNC, in case these get reset.
 			 */
-			mdb_fname_suffix_set(env, &fname, MDB_O_DATA);
-#ifdef _WIN32
-			len = OPEN_EXISTING;
-			env->me_mfd = CreateFileW(fname.mn_val, oflags,
-				FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, len,
-				mode | FILE_FLAG_WRITE_THROUGH, NULL);
-#else
-			oflags &= ~O_CREAT;
-			env->me_mfd = open(fname.mn_val, oflags | MDB_DSYNC, mode);
-#endif
-			if (env->me_mfd == INVALID_HANDLE_VALUE) {
-				rc = ErrCode();
+			rc = mdb_fopen(env, &fname, MDB_O_META, mode, &env->me_mfd);
+			if (rc)
 				goto leave;
-			}
 		}
 		DPRINTF(("opened dbenv %p", (void *) env));
 		if (excl > 0) {
@@ -10176,43 +10234,15 @@ mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags)
 	HANDLE newfd = INVALID_HANDLE_VALUE;
 
 	rc = mdb_fname_init(path, env->me_flags | MDB_NOLOCK, &fname);
-	if (rc)
-		return rc;
-	mdb_fname_suffix_set(env, &fname, MDB_O_DATA);
-
-	/* The destination path must exist, but the destination file must not.
-	 * We don't want the OS to cache the writes, since the source data is
-	 * already in the OS cache.
-	 */
-#ifdef _WIN32
-	newfd = CreateFileW(fname.mn_val, GENERIC_WRITE, 0, NULL, CREATE_NEW,
-				FILE_FLAG_NO_BUFFERING|FILE_FLAG_WRITE_THROUGH, NULL);
-#else
-	newfd = open(fname.mn_val, O_WRONLY|O_CREAT|O_EXCL, 0666);
-#endif
-	if (newfd == INVALID_HANDLE_VALUE) {
-		rc = ErrCode();
-		goto leave;
+	if (rc == MDB_SUCCESS) {
+		rc = mdb_fopen(env, &fname, MDB_O_COPY, 0666, &newfd);
+		mdb_fname_destroy(fname);
 	}
-
-	if (env->me_psize >= env->me_os_psize) {
-#ifdef F_NOCACHE	/* __APPLE__ */
-	(void) fcntl(newfd, F_NOCACHE, 1);
-#elif defined O_DIRECT
-	/* Set O_DIRECT if the file system supports it */
-	if ((rc = fcntl(newfd, F_GETFL)) != -1)
-		(void) fcntl(newfd, F_SETFL, rc | O_DIRECT);
-#endif
-	}
-
-	rc = mdb_env_copyfd2(env, newfd, flags);
-
-leave:
-	mdb_fname_destroy(fname);
-	if (newfd != INVALID_HANDLE_VALUE)
+	if (rc == MDB_SUCCESS) {
+		rc = mdb_env_copyfd2(env, newfd, flags);
 		if (close(newfd) < 0 && rc == MDB_SUCCESS)
 			rc = ErrCode();
-
+	}
 	return rc;
 }
 

From 04acac634a7b276332e2cc4d389b8d647a0a7fad Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 27 Sep 2016 07:03:38 +0200
Subject: [PATCH 264/504] ITS#8505 Set FD_CLOEXEC for me_mfd,env_copy as well

---
 libraries/liblmdb/mdb.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 0a7fcc02e8..27582ae745 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -4462,8 +4462,8 @@ enum mdb_fopen_type {
 	/* A comment in mdb_fopen() explains some O_* flag choices. */
 	MDB_O_RDONLY= O_RDONLY,                            /**< for RDONLY me_fd */
 	MDB_O_RDWR  = O_RDWR  |O_CREAT,                    /**< for me_fd */
-	MDB_O_META  = O_RDWR  |MDB_DSYNC,                  /**< for me_mfd */
-	MDB_O_COPY  = O_WRONLY|O_CREAT|O_EXCL,             /**< for #mdb_env_copy() */
+	MDB_O_META  = O_RDWR  |MDB_DSYNC     |MDB_CLOEXEC, /**< for me_mfd */
+	MDB_O_COPY  = O_WRONLY|O_CREAT|O_EXCL|MDB_CLOEXEC, /**< for #mdb_env_copy() */
 	/** Bitmask for open() flags in enum #mdb_fopen_type.  The other bits
 	 * distinguish otherwise-equal MDB_O_* constants from each other.
 	 */
@@ -4507,6 +4507,9 @@ mdb_fopen(const MDB_env *env, MDB_name *fname,
 	 *
 	 * The lockfile needs FD_CLOEXEC (close file descriptor on exec*())
 	 * to avoid the flock() issues noted under Caveats in lmdb.h.
+	 * Also set it for other filehandles which the user cannot get at
+	 * and close himself, which he may need after fork().  I.e. all but
+	 * me_fd, which programs do use via mdb_env_get_fd().
 	 */
 
 #ifdef _WIN32
@@ -4540,7 +4543,7 @@ mdb_fopen(const MDB_env *env, MDB_name *fname,
 		rc = ErrCode();
 #ifndef _WIN32
 	else {
-		if (which == MDB_O_LOCKS) {
+		if (which != MDB_O_RDONLY && which != MDB_O_RDWR) {
 			/* Set CLOEXEC if we could not pass it to open() */
 			if (!MDB_CLOEXEC && (flags = fcntl(fd, F_GETFD)) != -1)
 				(void) fcntl(fd, F_SETFD, flags | FD_CLOEXEC);

From 6355dade31e35af5527e9139a39fc1e3a5da04fd Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 27 Sep 2016 07:03:40 +0200
Subject: [PATCH 265/504] ITS#8505 Protect parent from fork()-pthread_exit()

---
 libraries/liblmdb/mdb.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 27582ae745..af365b3e25 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -4747,7 +4747,11 @@ mdb_env_reader_dest(void *ptr)
 {
 	MDB_reader *reader = ptr;
 
-	reader->mr_pid = 0;
+#ifndef _WIN32
+	if (reader->mr_pid == getpid()) /* catch pthread_exit() in child process */
+#endif
+		/* We omit the mutex, so do this atomically (i.e. skip mr_txnid) */
+		reader->mr_pid = 0;
 }
 
 #ifdef _WIN32

From 77845345ca9bf3854fd9da60a3e3b0527fa9c76a Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 27 Sep 2016 07:03:42 +0200
Subject: [PATCH 266/504] ITS#8505 Clarify fork() caveat, mdb_env_get_fd(),
 flock->fcntl.

---
 libraries/liblmdb/lmdb.h | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 30d586203a..319fcf62f1 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -97,11 +97,12 @@
  *	  transactions.  Each transaction belongs to one thread.  See below.
  *	  The #MDB_NOTLS flag changes this for read-only transactions.
  *
- *	- Use an MDB_env* in the process which opened it, without fork()ing.
+ *	- Use an MDB_env* in the process which opened it, not after fork().
  *
  *	- Do not have open an LMDB database twice in the same process at
  *	  the same time.  Not even from a plain open() call - close()ing it
- *	  breaks flock() advisory locking.
+ *	  breaks fcntl() advisory locking.  (It is OK to reopen it after
+ *	  fork() - exec*(), since the lockfile has FD_CLOEXEC set.)
  *
  *	- Avoid long-lived transactions.  Read transactions prevent
  *	  reuse of pages freed by newer write transactions, thus the
@@ -826,6 +827,10 @@ int  mdb_env_get_flags(MDB_env *env, unsigned int *flags);
 int  mdb_env_get_path(MDB_env *env, const char **path);
 
 	/** @brief Return the filedescriptor for the given environment.
+	 *
+	 * This function may be called after fork(), so the descriptor can be
+	 * closed before exec*().  Other LMDB file descriptors have FD_CLOEXEC.
+	 * (Until LMDB 0.9.18, only the lockfile had that.)
 	 *
 	 * @param[in] env An environment handle returned by #mdb_env_create()
 	 * @param[out] fd Address of a mdb_filehandle_t to contain the descriptor.

From f7e85d7804109a429dd6d24b19cc2148a7f43813 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 27 Sep 2016 07:03:45 +0200
Subject: [PATCH 267/504] Only set me_mfd if needed. Drop unused read access.

---
 libraries/liblmdb/mdb.c | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index af365b3e25..17f5a861cf 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1362,7 +1362,7 @@ typedef struct MDB_pgstate {
 struct MDB_env {
 	HANDLE		me_fd;		/**< The main data file */
 	HANDLE		me_lfd;		/**< The lock file */
-	HANDLE		me_mfd;			/**< just for writing the meta pages */
+	HANDLE		me_mfd;		/**< For writing and syncing the meta pages */
 #if defined(MDB_VL32) && defined(_WIN32)
 	HANDLE		me_fmh;		/**< File Mapping handle */
 #endif
@@ -4100,7 +4100,10 @@ mdb_env_write_meta(MDB_txn *txn)
 	len = sizeof(MDB_meta) - off;
 	off += (char *)mp - env->me_map;
 
-	/* Write to the SYNC fd */
+	/* Write to the SYNC fd unless MDB_NOSYNC/MDB_NOMETASYNC.
+	 * (me_mfd goes to the same file as me_fd, but writing to it
+	 * also syncs to disk.  Avoids a separate fdatasync() call.)
+	 */
 	mfd = (flags & (MDB_NOSYNC|MDB_NOMETASYNC)) ? env->me_fd : env->me_mfd;
 #ifdef _WIN32
 	{
@@ -4462,7 +4465,7 @@ enum mdb_fopen_type {
 	/* A comment in mdb_fopen() explains some O_* flag choices. */
 	MDB_O_RDONLY= O_RDONLY,                            /**< for RDONLY me_fd */
 	MDB_O_RDWR  = O_RDWR  |O_CREAT,                    /**< for me_fd */
-	MDB_O_META  = O_RDWR  |MDB_DSYNC     |MDB_CLOEXEC, /**< for me_mfd */
+	MDB_O_META  = O_WRONLY|MDB_DSYNC     |MDB_CLOEXEC, /**< for me_mfd */
 	MDB_O_COPY  = O_WRONLY|O_CREAT|O_EXCL|MDB_CLOEXEC, /**< for #mdb_env_copy() */
 	/** Bitmask for open() flags in enum #mdb_fopen_type.  The other bits
 	 * distinguish otherwise-equal MDB_O_* constants from each other.
@@ -4523,6 +4526,7 @@ mdb_fopen(const MDB_env *env, MDB_name *fname,
 		disp = OPEN_EXISTING;
 		break;
 	case MDB_O_META:			/* for writing metapages */
+		acc = GENERIC_WRITE;
 		disp = OPEN_EXISTING;
 		attrs = FILE_ATTRIBUTE_NORMAL|FILE_FLAG_WRITE_THROUGH;
 		break;
@@ -5339,9 +5343,7 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 	}
 
 	if ((rc = mdb_env_open2(env)) == MDB_SUCCESS) {
-		if (flags & (MDB_RDONLY|MDB_WRITEMAP)) {
-			env->me_mfd = env->me_fd;
-		} else {
+		if (!(flags & (MDB_RDONLY|MDB_WRITEMAP))) {
 			/* Synchronous fd for meta writes. Needed even with
 			 * MDB_NOSYNC/MDB_NOMETASYNC, in case these get reset.
 			 */
@@ -5447,7 +5449,7 @@ mdb_env_close0(MDB_env *env, int excl)
 		munmap(env->me_map, env->me_mapsize);
 #endif
 	}
-	if (env->me_mfd != env->me_fd && env->me_mfd != INVALID_HANDLE_VALUE)
+	if (env->me_mfd != INVALID_HANDLE_VALUE)
 		(void) close(env->me_mfd);
 	if (env->me_fd != INVALID_HANDLE_VALUE)
 		(void) close(env->me_fd);

From e58db7d5568c8ac91c0f1e6c64a58ba6f84b037d Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sat, 12 Nov 2016 23:11:20 +0000
Subject: [PATCH 268/504] More for ITS#8406

Revert excess cursor fixup
---
 libraries/liblmdb/mdb.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 17f5a861cf..026019eb2d 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -9198,9 +9198,11 @@ mdb_cursor_del0(MDB_cursor *mc)
 					}
 					if (mc->mc_db->md_flags & MDB_DUPSORT) {
 						MDB_node *node = NODEPTR(m3->mc_pg[m3->mc_top], m3->mc_ki[m3->mc_top]);
-						if (node->mn_flags & F_DUPDATA) {
+						/* If this node is a fake page, it needs to be reinited
+						 * because its data has moved.
+						 */
+						if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) {
 							mdb_xcursor_init1(m3, node);
-							m3->mc_xcursor->mx_cursor.mc_flags |= C_DEL;
 						}
 					}
 				}

From fa83b25ec536ac2642f8e0d3c6bf587008513ef4 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 29 Nov 2016 19:19:45 +0000
Subject: [PATCH 269/504] More for ITS#8406

xcursor fixup depends on init state
---
 libraries/liblmdb/mdb.c | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 026019eb2d..cb17f64021 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -9154,6 +9154,11 @@ mdb_cursor_del0(MDB_cursor *mc)
 			if (m3->mc_pg[mc->mc_top] == mp) {
 				if (m3->mc_ki[mc->mc_top] == ki) {
 					m3->mc_flags |= C_DEL;
+					if (mc->mc_db->md_flags & MDB_DUPSORT) {
+						/* Sub-cursor referred into dataset which is gone */
+						m3->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF);
+					}
+					continue;
 				} else if (m3->mc_ki[mc->mc_top] > ki) {
 					m3->mc_ki[mc->mc_top]--;
 				}
@@ -9199,10 +9204,14 @@ mdb_cursor_del0(MDB_cursor *mc)
 					if (mc->mc_db->md_flags & MDB_DUPSORT) {
 						MDB_node *node = NODEPTR(m3->mc_pg[m3->mc_top], m3->mc_ki[m3->mc_top]);
 						/* If this node is a fake page, it needs to be reinited
-						 * because its data has moved.
+						 * because its data has moved. But just reset mc_pg[0]
+						 * if the xcursor is already live.
 						 */
 						if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) {
-							mdb_xcursor_init1(m3, node);
+							if (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)
+								m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node);
+							else
+								mdb_xcursor_init1(m3, node);
 						}
 					}
 				}

From f3ab0d23a4470c187f971e6da4d63a306e125512 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Thu, 1 Dec 2016 21:17:42 +0100
Subject: [PATCH 270/504] Factor out refreshing sub-page pointers

---
 libraries/liblmdb/mdb.c | 88 +++++++++++++++++------------------------
 1 file changed, 36 insertions(+), 52 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index cb17f64021..7de15f6329 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1352,6 +1352,21 @@ typedef struct MDB_xcursor {
 	unsigned char mx_dbflag;
 } MDB_xcursor;
 
+	/** Check if there is an inited xcursor, so #XCURSOR_REFRESH() is proper */
+#define XCURSOR_INITED(mc) \
+	((mc)->mc_xcursor && ((mc)->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED))
+
+	/** Update sub-page pointer, if any, in \b mc->mc_xcursor.  Needed
+	 *	when the node which contains the sub-page may have moved.  Called
+	 *	with \b mp = mc->mc_pg[mc->mc_top], \b ki = mc->mc_ki[mc->mc_top].
+	 */
+#define XCURSOR_REFRESH(mc, mp, ki) do { \
+	MDB_page *xr_pg = (mp); \
+	MDB_node *xr_node = NODEPTR(xr_pg, ki); \
+	if ((xr_node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) \
+		(mc)->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(xr_node); \
+} while (0)
+
 	/** State of FreeDB old pages, stored in the MDB_env */
 typedef struct MDB_pgstate {
 	pgno_t		*mf_pghead;	/**< Reclaimed freeDB pages, or NULL before use */
@@ -1799,7 +1814,7 @@ mdb_cursor_chk(MDB_cursor *mc)
 	}
 	if (mc->mc_ki[i] >= NUMKEYS(mc->mc_pg[i]))
 		printf("ack!\n");
-	if (mc->mc_xcursor && (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) {
+	if (XCURSOR_INITED(mc)) {
 		node = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
 		if (((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) &&
 			mc->mc_xcursor->mx_cursor.mc_pg[0] != NODEDATA(node)) {
@@ -2679,14 +2694,8 @@ done:
 			if (m2 == mc) continue;
 			if (m2->mc_pg[mc->mc_top] == mp) {
 				m2->mc_pg[mc->mc_top] = np;
-				if ((mc->mc_db->md_flags & MDB_DUPSORT) &&
-					IS_LEAF(np) &&
-					(m2->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED))
-				{
-					MDB_node *leaf = NODEPTR(np, m2->mc_ki[mc->mc_top]);
-					if ((leaf->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA)
-						m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf);
-				}
+				if (XCURSOR_INITED(m2) && IS_LEAF(np))
+					XCURSOR_REFRESH(m2, np, m2->mc_ki[mc->mc_top]);
 			}
 		}
 	}
@@ -7654,11 +7663,8 @@ new_sub:
 				if (m3->mc_ki[i] >= mc->mc_ki[i] && insert_key) {
 					m3->mc_ki[i]++;
 				}
-				if (m3->mc_xcursor && (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) {
-					MDB_node *n2 = NODEPTR(mp, m3->mc_ki[i]);
-					if ((n2->mn_flags & (F_SUBDATA|F_DUPDATA)) == F_DUPDATA)
-						m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(n2);
-				}
+				if (XCURSOR_INITED(m3))
+					XCURSOR_REFRESH(m3, mp, m3->mc_ki[i]);
 			}
 		}
 	}
@@ -7709,9 +7715,7 @@ put_sub:
 						if (m2->mc_ki[i] == mc->mc_ki[i]) {
 							mdb_xcursor_init2(m2, mx, new_dupdata);
 						} else if (!insert_key && m2->mc_ki[i] < nkeys) {
-							MDB_node *n2 = NODEPTR(mp, m2->mc_ki[i]);
-							if ((n2->mn_flags & (F_SUBDATA|F_DUPDATA)) == F_DUPDATA)
-								m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(n2);
+							XCURSOR_REFRESH(m2, mp, m2->mc_ki[i]);
 						}
 					}
 				}
@@ -7816,13 +7820,12 @@ mdb_cursor_del(MDB_cursor *mc, unsigned int flags)
 						if (m2 == mc || m2->mc_snum < mc->mc_snum) continue;
 						if (!(m2->mc_flags & C_INITIALIZED)) continue;
 						if (m2->mc_pg[mc->mc_top] == mp) {
-							if (m2->mc_ki[mc->mc_top] == mc->mc_ki[mc->mc_top]) {
-								m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf);
-							} else {
-								MDB_node *n2 = NODEPTR(mp, m2->mc_ki[mc->mc_top]);
-								if (!(n2->mn_flags & F_SUBDATA))
-									m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(n2);
+							MDB_node *n2 = leaf;
+							if (m2->mc_ki[mc->mc_top] != mc->mc_ki[mc->mc_top]) {
+								n2 = NODEPTR(mp, m2->mc_ki[mc->mc_top]);
+								if (n2->mn_flags & F_SUBDATA) continue;
 							}
+							m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(n2);
 						}
 					}
 				}
@@ -8662,12 +8665,8 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft)
 					m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top];
 					m3->mc_ki[csrc->mc_top-1]++;
 				}
-				if (m3->mc_xcursor && (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) &&
-					IS_LEAF(mps)) {
-					MDB_node *node = NODEPTR(m3->mc_pg[csrc->mc_top], m3->mc_ki[csrc->mc_top]);
-					if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA)
-						m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node);
-				}
+				if (XCURSOR_INITED(m3) && IS_LEAF(mps))
+					XCURSOR_REFRESH(m3, m3->mc_pg[csrc->mc_top], m3->mc_ki[csrc->mc_top]);
 			}
 		} else
 		/* Adding on the right, bump others down */
@@ -8688,12 +8687,8 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft)
 					} else {
 						m3->mc_ki[csrc->mc_top]--;
 					}
-					if (m3->mc_xcursor && (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) &&
-						IS_LEAF(mps)) {
-						MDB_node *node = NODEPTR(m3->mc_pg[csrc->mc_top], m3->mc_ki[csrc->mc_top]);
-						if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA)
-							m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node);
-					}
+					if (XCURSOR_INITED(m3) && IS_LEAF(mps))
+						XCURSOR_REFRESH(m3, m3->mc_pg[csrc->mc_top], m3->mc_ki[csrc->mc_top]);
 				}
 			}
 		}
@@ -8894,12 +8889,8 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst)
 				m3->mc_ki[top-1] > csrc->mc_ki[top-1]) {
 				m3->mc_ki[top-1]--;
 			}
-			if (m3->mc_xcursor && (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) &&
-				IS_LEAF(psrc)) {
-				MDB_node *node = NODEPTR(m3->mc_pg[top], m3->mc_ki[top]);
-				if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA)
-					m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node);
-			}
+			if (XCURSOR_INITED(m3) && IS_LEAF(psrc))
+				XCURSOR_REFRESH(m3, m3->mc_pg[top], m3->mc_ki[top]);
 		}
 	}
 	{
@@ -9162,11 +9153,8 @@ mdb_cursor_del0(MDB_cursor *mc)
 				} else if (m3->mc_ki[mc->mc_top] > ki) {
 					m3->mc_ki[mc->mc_top]--;
 				}
-				if (m3->mc_xcursor && (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) {
-					MDB_node *node = NODEPTR(m3->mc_pg[mc->mc_top], m3->mc_ki[mc->mc_top]);
-					if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA)
-						m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node);
-				}
+				if (XCURSOR_INITED(m3))
+					XCURSOR_REFRESH(m3, m3->mc_pg[mc->mc_top], m3->mc_ki[mc->mc_top]);
 			}
 		}
 	}
@@ -9696,12 +9684,8 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 				m3->mc_ki[ptop] >= mc->mc_ki[ptop]) {
 				m3->mc_ki[ptop]++;
 			}
-			if (m3->mc_xcursor && (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) &&
-				IS_LEAF(mp)) {
-				MDB_node *node = NODEPTR(m3->mc_pg[mc->mc_top], m3->mc_ki[mc->mc_top]);
-				if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA)
-					m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node);
-			}
+			if (XCURSOR_INITED(m3) && IS_LEAF(mp))
+				XCURSOR_REFRESH(m3, m3->mc_pg[mc->mc_top], m3->mc_ki[mc->mc_top]);
 		}
 	}
 	DPRINTF(("mp left: %d, rp left: %d", SIZELEFT(mp), SIZELEFT(rp)));

From a70200f15f06397ef9029ca26fabc925ce396ead Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Fri, 2 Dec 2016 05:22:11 +0100
Subject: [PATCH 271/504] Fix mdb_env_open() with MDB_VL32

Init me_rpmutex independently of MDB_NOLOCK.
Plug leaks on mdb_env_open() failure.
Tweak mdb_env_close0() to handle the rearranged mdb_env_open().
---
 libraries/liblmdb/mdb.c | 42 ++++++++++++++++++++++++++---------------
 1 file changed, 27 insertions(+), 15 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 7de15f6329..ea0b8ef21f 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5245,13 +5245,6 @@ mdb_env_setup_locks(MDB_env *env, MDB_name *fname, int mode, int *excl)
 	env->me_wmutex->semnum = 1;
 	env->me_rmutex->locked = &env->me_txns->mti_rlocked;
 	env->me_wmutex->locked = &env->me_txns->mti_wlocked;
-#endif
-#ifdef MDB_VL32
-#ifdef _WIN32
-	env->me_rpmutex = CreateMutex(NULL, FALSE, NULL);
-#else
-	pthread_mutex_init(&env->me_rpmutex, NULL);
-#endif
 #endif
 
 	return MDB_SUCCESS;
@@ -5299,6 +5292,21 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 	if (rc)
 		return rc;
 
+#ifdef MDB_VL32
+#ifdef _WIN32
+	env->me_rpmutex = CreateMutex(NULL, FALSE, NULL);
+	if (!env->me_rpmutex) {
+		rc = ErrCode();
+		goto leave;
+	}
+#else
+	rc = pthread_mutex_init(&env->me_rpmutex, NULL);
+	if (rc)
+		goto leave;
+#endif
+#endif
+	flags |= MDB_ENV_ACTIVE;	/* tell mdb_env_close0() to clean up */
+
 	if (flags & MDB_RDONLY) {
 		/* silently ignore WRITEMAP when we're only getting read access */
 		flags &= ~MDB_WRITEMAP;
@@ -5307,8 +5315,13 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 			  (env->me_dirty_list = calloc(MDB_IDL_UM_SIZE, sizeof(MDB_ID2)))))
 			rc = ENOMEM;
 	}
+
+	env->me_flags = flags;
+	if (rc)
+		goto leave;
+
 #ifdef MDB_VL32
-	if (!rc) {
+	{
 		env->me_rpages = malloc(MDB_ERPAGE_SIZE * sizeof(MDB_ID3));
 		if (!env->me_rpages) {
 			rc = ENOMEM;
@@ -5318,9 +5331,6 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 		env->me_rpcheck = MDB_ERPAGE_SIZE/2;
 	}
 #endif
-	env->me_flags = flags |= MDB_ENV_ACTIVE;
-	if (rc)
-		goto leave;
 
 	env->me_path = strdup(path);
 	env->me_dbxs = calloc(env->me_maxdbs, sizeof(MDB_dbx));
@@ -5429,11 +5439,13 @@ mdb_env_close0(MDB_env *env, int excl)
 #ifdef MDB_VL32
 	if (env->me_txn0 && env->me_txn0->mt_rpages)
 		free(env->me_txn0->mt_rpages);
-	{ unsigned int x;
-		for (x=1; x<=env->me_rpages[0].mid; x++)
-		munmap(env->me_rpages[x].mptr, env->me_rpages[x].mcnt * env->me_psize);
+	if (env->me_rpages) {
+		MDB_ID3L el = env->me_rpages;
+		unsigned int x;
+		for (x=1; x<=el[0].mid; x++)
+			munmap(el[x].mptr, el[x].mcnt * env->me_psize);
+		free(el);
 	}
-	free(env->me_rpages);
 #endif
 	free(env->me_txn0);
 	mdb_midl_free(env->me_free_pgs);

From 08e4684d37b45cff97ea0ba17573165a63fb622c Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Fri, 2 Dec 2016 06:20:29 +0100
Subject: [PATCH 272/504] MDB_VL32 mdb_page_get(): Set MDB_TXN_ERROR on
 failure.

---
 libraries/liblmdb/mdb.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index ea0b8ef21f..4a79ae13f0 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -6150,8 +6150,10 @@ mapped:
 	{
 #ifdef MDB_VL32
 		int rc = mdb_rpage_get(txn, pgno, &p);
-		if (rc)
+		if (rc) {
+			txn->mt_flags |= MDB_TXN_ERROR;
 			return rc;
+		}
 #else
 		MDB_env *env = txn->mt_env;
 		p = (MDB_page *)(env->me_map + env->me_psize * pgno);

From 72f875b48a0f9c588e4f26828f18c547525a507d Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Wed, 7 Dec 2016 18:55:21 +0100
Subject: [PATCH 273/504] Note functions which must set MDB_TXN_ERROR on
 failure

Other functions depend on them to do so.
For mdb_node_read(), instead remove such a dependence.
---
 libraries/liblmdb/mdb.c | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 4a79ae13f0..a0e65d9449 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1899,6 +1899,7 @@ mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b)
 
 /** Allocate memory for a page.
  * Re-use old malloc'd pages first for singletons, otherwise just malloc.
+ * Set #MDB_TXN_ERROR on failure.
  */
 static MDB_page *
 mdb_page_malloc(MDB_txn *txn, unsigned num)
@@ -2310,7 +2311,7 @@ mdb_page_dirty(MDB_txn *txn, MDB_page *mp)
 }
 
 /** Allocate page numbers and memory for writing.  Maintain me_pglast,
- * me_pghead and mt_next_pgno.
+ * me_pghead and mt_next_pgno.  Set #MDB_TXN_ERROR on failure.
  *
  * If there are free pages available from older transactions, they
  * are re-used first. Otherwise allocate a new page at mt_next_pgno.
@@ -2442,7 +2443,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
 		np = m2.mc_pg[m2.mc_top];
 		leaf = NODEPTR(np, m2.mc_ki[m2.mc_top]);
 		if ((rc = mdb_node_read(&m2, leaf, &data)) != MDB_SUCCESS)
-			return rc;
+			goto fail;
 
 		idl = (MDB_ID *) data.mv_data;
 		i = idl[0];
@@ -2606,6 +2607,7 @@ mdb_page_unspill(MDB_txn *txn, MDB_page *mp, MDB_page **ret)
 }
 
 /** Touch a page: make it dirty and re-insert into tree with updated pgno.
+ * Set #MDB_TXN_ERROR on failure.
  * @param[in] mc cursor pointing to the page to be touched
  * @return 0 on success, non-zero on failure.
  */
@@ -5788,7 +5790,9 @@ mdb_cursor_pop(MDB_cursor *mc)
 	}
 }
 
-/** Push a page onto the top of the cursor's stack. */
+/** Push a page onto the top of the cursor's stack.
+ * Set #MDB_TXN_ERROR on failure.
+ */
 static int
 mdb_cursor_push(MDB_cursor *mc, MDB_page *mp)
 {
@@ -6096,6 +6100,7 @@ ok:
 #endif
 
 /** Find the address of the page corresponding to a given page number.
+ * Set #MDB_TXN_ERROR on failure.
  * @param[in] mc the cursor accessing the page.
  * @param[in] pgno the page number for the page to retrieve.
  * @param[out] ret address of a pointer where the page's address will be stored.
@@ -7884,6 +7889,7 @@ fail:
 }
 
 /** Allocate and initialize new pages for a database.
+ * Set #MDB_TXN_ERROR on failure.
  * @param[in] mc a cursor on the database being added to.
  * @param[in] flags flags defining what type of page is being allocated.
  * @param[in] num the number of pages to allocate. This is usually 1,
@@ -7969,6 +7975,7 @@ mdb_branch_size(MDB_env *env, MDB_val *key)
 }
 
 /** Add a node to the page pointed to by the cursor.
+ * Set #MDB_TXN_ERROR on failure.
  * @param[in] mc The cursor for this operation.
  * @param[in] indx The index on the page where the new node should be added.
  * @param[in] key The key for the new node.
@@ -8460,6 +8467,7 @@ mdb_cursor_dbi(MDB_cursor *mc)
 }
 
 /** Replace the key for a branch node with a new key.
+ * Set #MDB_TXN_ERROR on failure.
  * @param[in] mc Cursor pointing to the node to operate on.
  * @param[in] key The new key to use.
  * @return 0 on success, non-zero on failure.
@@ -9289,6 +9297,7 @@ mdb_del0(MDB_txn *txn, MDB_dbi dbi,
 }
 
 /** Split a page and insert a new node.
+ * Set #MDB_TXN_ERROR on failure.
  * @param[in,out] mc Cursor pointing to the page and desired insertion index.
  * The cursor will be updated to point to the actual page and index where
  * the node got inserted after the split.

From 3dda2bfa4a33e7f22ab92dafc501411c90647e16 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Wed, 7 Dec 2016 19:04:19 +0100
Subject: [PATCH 274/504] doxygen cleanup

---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index a0e65d9449..e43e8dcf42 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -10866,7 +10866,7 @@ mdb_reader_check(MDB_env *env, int *dead)
 	return env->me_txns ? mdb_reader_check0(env, 0, dead) : MDB_SUCCESS;
 }
 
-/** As #mdb_reader_check(). rlocked = <caller locked the reader mutex>. */
+/** As #mdb_reader_check(). \b rlocked is set if caller locked #me_rmutex. */
 static int ESECT
 mdb_reader_check0(MDB_env *env, int rlocked, int *dead)
 {

From ef066598b5e4c0eaab29c858451569439cb6b5d6 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Wed, 7 Dec 2016 19:06:11 +0100
Subject: [PATCH 275/504] MDB_CP_COMPACT comments

---
 libraries/liblmdb/mdb.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index e43e8dcf42..b057829ac5 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -9765,7 +9765,10 @@ typedef struct mdb_copy {
 	HANDLE mc_fd;
 	int mc_toggle;			/**< Buffer number in provider */
 	int mc_new;				/**< (0-2 buffers to write) | (#MDB_EOF at end) */
-	volatile int mc_error;	/**< Error code, never cleared if set */
+	/** Error code.  Never cleared if set.  Both threads can set nonzero
+	 *	to fail the copy.  Not mutex-protected, LMDB expects atomic int.
+	 */
+	volatile int mc_error;
 } mdb_copy;
 
 	/** Dedicated writer thread for compacting copy. */
@@ -9850,7 +9853,11 @@ mdb_env_cthr_toggle(mdb_copy *my, int adjust)
 	return my->mc_error;
 }
 
-	/** Depth-first tree traversal for compacting copy. */
+	/** Depth-first tree traversal for compacting copy.
+	 * @param[in] my control structure.
+	 * @param[in,out] pg database root.
+	 * @param[in] flags includes #F_DUPDATA if it is a sorted-duplicate sub-DB.
+	 */
 static int ESECT
 mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags)
 {

From ffd13db3f84cef20f486e73c465d674e6e1500bd Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Fri, 9 Dec 2016 00:03:36 +0100
Subject: [PATCH 276/504] Cleanup: Add flag DB_DUPDATA, drop DB_DIRTY hack

---
 libraries/liblmdb/mdb.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index b057829ac5..27e63ee5ad 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1214,11 +1214,12 @@ struct MDB_txn {
  *	@ingroup internal
  * @{
  */
-#define DB_DIRTY	0x01		/**< DB was modified or is DUPSORT data */
+#define DB_DIRTY	0x01		/**< DB was written in this txn */
 #define DB_STALE	0x02		/**< Named-DB record is older than txnID */
 #define DB_NEW		0x04		/**< Named-DB handle opened in this txn */
 #define DB_VALID	0x08		/**< DB handle is valid, see also #MDB_VALID */
 #define DB_USRVALID	0x10		/**< As #DB_VALID, but not set for #FREE_DBI */
+#define DB_DUPDATA	0x20		/**< DB is #MDB_DUPSORT data */
 /** @} */
 	/** In write txns, array of cursors for each DB */
 	MDB_cursor	**mt_cursors;
@@ -7230,7 +7231,8 @@ mdb_cursor_touch(MDB_cursor *mc)
 {
 	int rc = MDB_SUCCESS;
 
-	if (mc->mc_dbi >= CORE_DBS && !(*mc->mc_dbflag & DB_DIRTY)) {
+	if (mc->mc_dbi >= CORE_DBS && !(*mc->mc_dbflag & (DB_DIRTY|DB_DUPDATA))) {
+		/* Touch DB record of named DB */
 		MDB_cursor mc2;
 		MDB_xcursor mcx;
 		if (TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbi))
@@ -8286,7 +8288,7 @@ mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node)
 	}
 	DPRINTF(("Sub-db -%u root page %"Yu, mx->mx_cursor.mc_dbi,
 		mx->mx_db.md_root));
-	mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DIRTY; /* DB_DIRTY guides mdb_cursor_touch */
+	mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DUPDATA;
 	if (NEED_CMP_CLONG(mx->mx_dbx.md_cmp, mx->mx_db.md_pad))
 		mx->mx_dbx.md_cmp = mdb_cmp_clong;
 }
@@ -8310,7 +8312,7 @@ mdb_xcursor_init2(MDB_cursor *mc, MDB_xcursor *src_mx, int new_dupdata)
 		mx->mx_cursor.mc_top = 0;
 		mx->mx_cursor.mc_flags |= C_INITIALIZED;
 		mx->mx_cursor.mc_ki[0] = 0;
-		mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DIRTY; /* DB_DIRTY guides mdb_cursor_touch */
+		mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DUPDATA;
 #if UINT_MAX < MDB_SIZE_MAX	/* matches mdb_xcursor_init1:NEED_CMP_CLONG() */
 		mx->mx_dbx.md_cmp = src_mx->mx_dbx.md_cmp;
 #endif

From e1be73c771e3ac8dfc7528c0825c56eb8ed295a6 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 10 Dec 2016 09:16:17 +0100
Subject: [PATCH 277/504] ITS#8542 mdb_dbi_open(): Protect mainDB cursors

---
 libraries/liblmdb/mdb.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 27e63ee5ad..7974261437 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -10515,7 +10515,8 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db
 		memset(&dummy, 0, sizeof(dummy));
 		dummy.md_root = P_INVALID;
 		dummy.md_flags = flags & PERSISTENT_FLAGS;
-		rc = mdb_cursor_put(&mc, &key, &data, F_SUBDATA);
+		WITH_CURSOR_TRACKING(mc,
+			rc = mdb_cursor_put(&mc, &key, &data, F_SUBDATA));
 		dbflag |= DB_DIRTY;
 	}
 

From c0ff9a267ae2a943c99452fae0159d77deb46562 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 10 Dec 2016 17:17:28 +0100
Subject: [PATCH 278/504] MDB_VL32 cleanup: MDB_env.mm_mapsize type

---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 7974261437..f70e9f0a6b 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1129,7 +1129,7 @@ typedef struct MDB_meta {
 #else
 	void		*mm_address;		/**< address for fixed mapping */
 #endif
-	pgno_t		mm_mapsize;			/**< size of mmap region */
+	mdb_size_t	mm_mapsize;			/**< size of mmap region */
 	MDB_db		mm_dbs[CORE_DBS];	/**< first is free space, 2nd is main db */
 	/** The size of pages used in this DB */
 #define	mm_psize	mm_dbs[FREE_DBI].md_pad

From 1fb0822b408931f7e0d7d0a868b833a89c83b27f Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 10 Dec 2016 21:11:12 +0100
Subject: [PATCH 279/504] ITS#8355 cleanup

Drop ~C_EOF, pointless after 8c215aa970215a58ee0df458813c0405ad27a6e9
---
 libraries/liblmdb/mdb.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index f70e9f0a6b..edfb034e06 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -6714,8 +6714,6 @@ mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
 	} else
 		mc->mc_ki[mc->mc_top]--;
 
-	mc->mc_flags &= ~C_EOF;
-
 	DPRINTF(("==> cursor points to page %"Yu" with %u keys, key index %u",
 	    mdb_dbg_pgno(mp), NUMKEYS(mp), mc->mc_ki[mc->mc_top]));
 

From d78c80d902e3b508b6d118c975a6c6a8d9782f40 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 10 Dec 2016 21:42:39 +0100
Subject: [PATCH 280/504] Clean up and comment C_UNTRACK

Don't use it as a "cursor is tracked" hint in mdb_pages_xkeep().
It's been harmless so far, but would break after mdb_cursor_copy().
Checking m0 directly short-circuits better anyway.
---
 libraries/liblmdb/mdb.c | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index edfb034e06..5a3a829629 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2094,13 +2094,9 @@ mdb_pages_xkeep(MDB_cursor *mc, unsigned pflags, int all)
 	unsigned i, j;
 	int rc = MDB_SUCCESS, level;
 
-	/* Mark pages seen by cursors */
-	if (mc->mc_flags & C_UNTRACK)
-		mc = NULL;				/* will find mc in mt_cursors */
-	for (i = txn->mt_numdbs;; mc = txn->mt_cursors[--i]) {
-		for (; mc; mc=mc->mc_next) {
-			if (!(mc->mc_flags & C_INITIALIZED))
-				continue;
+	/* Mark pages seen by cursors: First m0, then tracked cursors */
+	for (i = txn->mt_numdbs;; ) {
+		if (mc->mc_flags & C_INITIALIZED) {
 			for (m3 = mc;; m3 = &mx->mx_cursor) {
 				mp = NULL;
 				for (j=0; j<m3->mc_snum; j++) {
@@ -2119,10 +2115,13 @@ mdb_pages_xkeep(MDB_cursor *mc, unsigned pflags, int all)
 					break;
 			}
 		}
-		if (i == 0)
-			break;
+		mc = mc->mc_next;
+		for (; !mc || mc == m0; mc = txn->mt_cursors[--i])
+			if (i == 0)
+				goto mark_done;
 	}
 
+mark_done:
 	if (all) {
 		/* Mark dirty root pages */
 		for (i=0; i<txn->mt_numdbs; i++) {
@@ -8442,7 +8441,10 @@ mdb_cursor_close(MDB_cursor *mc)
 		MDB_CURSOR_UNREF(mc, 0);
 	}
 	if (mc && !mc->mc_backup) {
-		/* remove from txn, if tracked */
+		/* Remove from txn, if tracked.
+		 * A read-only txn (!C_UNTRACK) may have been freed already,
+		 * so do not peek inside it.  Only write txns track cursors.
+		 */
 		if ((mc->mc_flags & C_UNTRACK) && mc->mc_txn->mt_cursors) {
 			MDB_cursor **prev = &mc->mc_txn->mt_cursors[mc->mc_dbi];
 			while (*prev && *prev != mc) prev = &(*prev)->mc_next;
@@ -9287,7 +9289,6 @@ mdb_del0(MDB_txn *txn, MDB_dbi dbi,
 		 * run out of space, triggering a split. We need this
 		 * cursor to be consistent until the end of the rebalance.
 		 */
-		mc.mc_flags |= C_UNTRACK;
 		mc.mc_next = txn->mt_cursors[dbi];
 		txn->mt_cursors[dbi] = &mc;
 		rc = mdb_cursor_del(&mc, flags);

From 65e95ffccf512dd1514083bd5768f1595560eb53 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 10 Dec 2016 22:00:31 +0100
Subject: [PATCH 281/504] ITS#7377 Catch mdb_cursor_sibling() error

---
 libraries/liblmdb/mdb.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 5a3a829629..42452c2894 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -9548,7 +9548,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 			} else {
 				/* find right page's left sibling */
 				mc->mc_ki[ptop] = mn.mc_ki[ptop];
-				mdb_cursor_sibling(mc, 0);
+				rc = mdb_cursor_sibling(mc, 0);
 			}
 		}
 	} else {
@@ -9557,6 +9557,8 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 		mn.mc_top++;
 	}
 	if (rc != MDB_SUCCESS) {
+		if (rc == MDB_NOTFOUND) /* improper mdb_cursor_sibling() result */
+			rc = MDB_PROBLEM;
 		goto done;
 	}
 	if (nflags & MDB_APPEND) {

From 3e7a8e26e6a06ef34fcb460b608c6e65f688b9a6 Mon Sep 17 00:00:00 2001
From: Lorenz Bauer <lmb@cloudflare.com>
Date: Thu, 20 Oct 2016 09:51:22 +0200
Subject: [PATCH 282/504] ITS#8504 mdb_env_copyfd2(): Don't abort on SIGPIPE

Return EPIPE instead.
---
 libraries/liblmdb/mdb.c | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 42452c2894..b87e576fda 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -160,6 +160,7 @@ typedef SSIZE_T	ssize_t;
 
 #ifndef _WIN32
 #include <pthread.h>
+#include <signal.h>
 #ifdef MDB_USE_POSIX_SEM
 # define MDB_USE_HASH		1
 #include <semaphore.h>
@@ -9786,10 +9787,17 @@ mdb_env_copythr(void *arg)
 #define DO_WRITE(rc, fd, ptr, w2, len)	rc = WriteFile(fd, ptr, w2, &len, NULL)
 #else
 	int len;
+	sigset_t set;
 #define DO_WRITE(rc, fd, ptr, w2, len)	len = write(fd, ptr, w2); rc = (len >= 0)
+
+	sigemptyset(&set);
+	sigaddset(&set, SIGPIPE);
 #endif
 
 	pthread_mutex_lock(&my->mc_mutex);
+#ifndef _WIN32
+	my->mc_error = pthread_sigmask(SIG_BLOCK, &set, NULL);
+#endif
 	for(;;) {
 		while (!my->mc_new)
 			pthread_cond_wait(&my->mc_cond, &my->mc_mutex);
@@ -9803,6 +9811,12 @@ again:
 			DO_WRITE(rc, my->mc_fd, ptr, wsize, len);
 			if (!rc) {
 				rc = ErrCode();
+#ifndef _WIN32
+				if (rc == EPIPE) {
+					int tmp;
+					sigwait(&set, &tmp);
+				}
+#endif
 				break;
 			} else if (len > 0) {
 				rc = MDB_SUCCESS;

From e539654051a6300997be7420e67b31d08c87aa90 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Thu, 20 Oct 2016 09:51:22 +0200
Subject: [PATCH 283/504] ITS#8504 Fix prev commit: mc_error, #ifdef SIGPIPE

Never clear mc_error, we could lose a failure in the other thread.
---
 libraries/liblmdb/mdb.c | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index b87e576fda..9e0f907588 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -9787,17 +9787,17 @@ mdb_env_copythr(void *arg)
 #define DO_WRITE(rc, fd, ptr, w2, len)	rc = WriteFile(fd, ptr, w2, &len, NULL)
 #else
 	int len;
-	sigset_t set;
 #define DO_WRITE(rc, fd, ptr, w2, len)	len = write(fd, ptr, w2); rc = (len >= 0)
-
+#ifdef SIGPIPE
+	sigset_t set;
 	sigemptyset(&set);
 	sigaddset(&set, SIGPIPE);
+	if ((rc = pthread_sigmask(SIG_BLOCK, &set, NULL)) != 0)
+		my->mc_error = rc;
+#endif
 #endif
 
 	pthread_mutex_lock(&my->mc_mutex);
-#ifndef _WIN32
-	my->mc_error = pthread_sigmask(SIG_BLOCK, &set, NULL);
-#endif
 	for(;;) {
 		while (!my->mc_new)
 			pthread_cond_wait(&my->mc_cond, &my->mc_mutex);
@@ -9811,8 +9811,11 @@ again:
 			DO_WRITE(rc, my->mc_fd, ptr, wsize, len);
 			if (!rc) {
 				rc = ErrCode();
-#ifndef _WIN32
+#if defined(SIGPIPE) && !defined(_WIN32)
 				if (rc == EPIPE) {
+					/* Collect the pending SIGPIPE, otherwise at least OS X
+					 * gives it to the process on thread-exit (ITS#8504).
+					 */
 					int tmp;
 					sigwait(&set, &tmp);
 				}

From 72c893fc82899b4e4d4fb506116ba109ee66032a Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Thu, 15 Dec 2016 22:12:45 +0100
Subject: [PATCH 284/504] Mention MDB_PREV_MULTIPLE along with
 MDB_NEXT_MULTIPLE

---
 libraries/liblmdb/lmdb.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 319fcf62f1..146bf3b5be 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -1140,8 +1140,9 @@ int  mdb_txn_renew(MDB_txn *txn);
 	 *		This flag may only be used in combination with #MDB_DUPSORT. This option
 	 *		tells the library that the data items for this database are all the same
 	 *		size, which allows further optimizations in storage and retrieval. When
-	 *		all data items are the same size, the #MDB_GET_MULTIPLE and #MDB_NEXT_MULTIPLE
-	 *		cursor operations may be used to retrieve multiple items at once.
+	 *		all data items are the same size, the #MDB_GET_MULTIPLE, #MDB_NEXT_MULTIPLE
+	 *		and #MDB_PREV_MULTIPLE cursor operations may be used to retrieve multiple
+	 *		items at once.
 	 *	<li>#MDB_INTEGERDUP
 	 *		This option specifies that duplicate data items are binary integers,
 	 *		similar to #MDB_INTEGERKEY keys.

From be94a7565bfb553bb1a0ab71ddd602d48ffcaa0f Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Wed, 21 Dec 2016 16:33:47 +0100
Subject: [PATCH 285/504] Doxygen fixes. Use DISTRIBUTE_GROUP_DOC.

- DISTRIBUTE_GROUP_DOC makes doxygen give several fields the
  same doc: mn_hi + mn_lo in MDB_node.
- With mdb_mutex_t + mdb_mutexref_t, instead split them up.
- Don't hide a doxygen #name inside double quotes.
---
 libraries/liblmdb/Doxyfile |  2 +-
 libraries/liblmdb/mdb.c    | 32 +++++++++++++++++---------------
 2 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/libraries/liblmdb/Doxyfile b/libraries/liblmdb/Doxyfile
index 5047c0bb1f..5ca2cfe8f6 100644
--- a/libraries/liblmdb/Doxyfile
+++ b/libraries/liblmdb/Doxyfile
@@ -253,7 +253,7 @@ IDL_PROPERTY_SUPPORT   = YES
 # member in the group (if any) for the other members of the group. By default
 # all members of a group must be documented explicitly.
 
-DISTRIBUTE_GROUP_DOC   = NO
+DISTRIBUTE_GROUP_DOC   = YES
 
 # Set the SUBGROUPING tag to YES (the default) to allow class member groups of
 # the same type (for instance a group of public functions) to be put as a
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 9e0f907588..5b799e112b 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -417,15 +417,15 @@ mdb_sem_wait(mdb_mutexref_t sem)
 #define mdb_mutex_consistent(mutex)	0
 
 #else	/* MDB_USE_POSIX_MUTEX: */
-	/** Shared mutex/semaphore as it is stored (mdb_mutex_t), and as
-	 *	local variables keep it (mdb_mutexref_t).
+	/** Shared mutex/semaphore as the original is stored.
 	 *
-	 *	An mdb_mutex_t can be assigned to an mdb_mutexref_t.  They can
-	 *	be the same, or an array[size 1] and a pointer.
-	 *	@{
+	 *	Not for copies.  Instead it can be assigned to an #mdb_mutexref_t.
+	 *	When mdb_mutexref_t is a pointer and mdb_mutex_t is not, then it
+	 *	is array[size 1] so it can be assigned to the pointer.
 	 */
-typedef pthread_mutex_t mdb_mutex_t[1], *mdb_mutexref_t;
-	/*	@} */
+typedef pthread_mutex_t mdb_mutex_t[1];
+	/** Reference to an #mdb_mutex_t */
+typedef pthread_mutex_t *mdb_mutexref_t;
 	/** Lock the reader or writer mutex.
 	 *	Returns 0 or a code to give #mdb_mutex_failed(), as in #LOCK_MUTEX().
 	 */
@@ -463,7 +463,7 @@ typedef pthread_mutex_t mdb_mutex_t[1], *mdb_mutexref_t;
 
 #define	Z	MDB_FMT_Z	/**< printf/scanf format modifier for size_t */
 #define	Yu	MDB_PRIy(u)	/**< printf format for #mdb_size_t */
-#define	Yd	MDB_PRIy(d)	/**< printf format for "signed #mdb_size_t" */
+#define	Yd	MDB_PRIy(d)	/**< printf format for 'signed #mdb_size_t' */
 
 #if defined(_WIN32) || defined(MDB_USE_POSIX_SEM)
 #define MNAME_LEN	32
@@ -973,19 +973,21 @@ typedef struct MDB_page {
 	/** Header for a single key/data pair within a page.
 	 * Used in pages of type #P_BRANCH and #P_LEAF without #P_LEAF2.
 	 * We guarantee 2-byte alignment for 'MDB_node's.
+	 *
+	 * #mn_lo and #mn_hi are used for data size on leaf nodes, and for child
+	 * pgno on branch nodes.  On 64 bit platforms, #mn_flags is also used
+	 * for pgno.  (Branch nodes have no flags).  Lo and hi are in host byte
+	 * order in case some accesses can be optimized to 32-bit word access.
 	 */
 typedef struct MDB_node {
-	/** lo and hi are used for data size on leaf nodes and for
-	 * child pgno on branch nodes. On 64 bit platforms, flags
-	 * is also used for pgno. (Branch nodes have no flags).
-	 * They are in host byte order in case that lets some
-	 * accesses be optimized into a 32-bit word access.
-	 */
+	/** part of data size or pgno
+	 *	@{ */
 #if BYTE_ORDER == LITTLE_ENDIAN
-	unsigned short	mn_lo, mn_hi;	/**< part of data size or pgno */
+	unsigned short	mn_lo, mn_hi;
 #else
 	unsigned short	mn_hi, mn_lo;
 #endif
+	/** @} */
 /** @defgroup mdb_node Node Flags
  *	@ingroup internal
  *	Flags for node headers.

From 4bc270a2cb5d5a4ef1f01c70faa2bdfe5019ef16 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Wed, 21 Dec 2016 21:40:14 +0100
Subject: [PATCH 286/504] More MDB_node doc

---
 libraries/liblmdb/mdb.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 5b799e112b..5ca0e0ae0f 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -978,6 +978,11 @@ typedef struct MDB_page {
 	 * pgno on branch nodes.  On 64 bit platforms, #mn_flags is also used
 	 * for pgno.  (Branch nodes have no flags).  Lo and hi are in host byte
 	 * order in case some accesses can be optimized to 32-bit word access.
+	 *
+	 * Leaf node flags describe node contents.  #F_BIGDATA says the node's
+	 * data part is the page number of an overflow page with actual data.
+	 * #F_DUPDATA and #F_SUBDATA can be combined giving duplicate data in
+	 * a sub-page/sub-database, and named databases (just #F_SUBDATA).
 	 */
 typedef struct MDB_node {
 	/** part of data size or pgno

From 2e3eaf2ce26bf643d52c4c98b77b9d9bc71ed2c1 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 28 Dec 2016 18:30:19 +0000
Subject: [PATCH 287/504] ITS#8554 kFreeBSD is like BSD

Doesn't have POSIX robust mutexes - GNU userland on BSD kernel
---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 5ca0e0ae0f..3e7af11886 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -149,7 +149,7 @@ typedef SSIZE_T	ssize_t;
 #include <resolv.h>	/* defines BYTE_ORDER on HPUX and Solaris */
 #endif
 
-#if defined(__APPLE__) || defined (BSD)
+#if defined(__APPLE__) || defined (BSD) || defined(__FreeBSD_kernel__)
 # if !(defined(MDB_USE_POSIX_MUTEX) || defined(MDB_USE_POSIX_SEM))
 # define MDB_USE_SYSV_SEM	1
 # endif

From 59ac317d2a4a81613e3e1dbc17fd2d3ec4205ba1 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Fri, 6 Jan 2017 19:48:58 +0000
Subject: [PATCH 288/504] ITS#8558 fix mdb_load with escaped plaintext

---
 libraries/liblmdb/mdb_load.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c
index 73dfe8cfd9..c421a6980f 100644
--- a/libraries/liblmdb/mdb_load.c
+++ b/libraries/liblmdb/mdb_load.c
@@ -245,7 +245,8 @@ badend:
 					c2 += 2;
 				}
 			} else {
-				c1++; c2++;
+				/* copies are redundant when no escapes were used */
+				*c1++ = *c2++;
 			}
 		}
 	} else {

From d84dee516fa4cca41b5234e95c2105eb4737dfb3 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 11 Jan 2017 09:51:43 +0000
Subject: [PATCH 289/504] ITS#8557 fix mdb_cursor_last

Optimize mdb_page_search_root(PS_LAST) when cursor is already near
last position, ignoring C_EOF flag for now.
---
 libraries/liblmdb/mdb.c | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 3e7af11886..c30a96f4c6 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -6204,8 +6204,17 @@ mdb_page_search_root(MDB_cursor *mc, MDB_val *key, int flags)
 
 		if (flags & (MDB_PS_FIRST|MDB_PS_LAST)) {
 			i = 0;
-			if (flags & MDB_PS_LAST)
+			if (flags & MDB_PS_LAST) {
 				i = NUMKEYS(mp) - 1;
+				/* if already init'd, see if we're already in right place */
+				if (mc->mc_flags & C_INITIALIZED) {
+					if (mc->mc_ki[mc->mc_top] == i) {
+						mp = mc->mc_pg[mc->mc_top];
+						mc->mc_top = mc->mc_snum++;
+						goto ready;
+					}
+				}
+			}
 		} else {
 			int	 exact;
 			node = mdb_node_search(mc, key, &exact);
@@ -6231,6 +6240,7 @@ mdb_page_search_root(MDB_cursor *mc, MDB_val *key, int flags)
 		if ((rc = mdb_cursor_push(mc, mp)))
 			return rc;
 
+ready:
 		if (flags & MDB_PS_MODIFY) {
 			if ((rc = mdb_page_touch(mc)) != 0)
 				return rc;
@@ -7009,16 +7019,13 @@ mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data)
 		mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF);
 	}
 
-	if (!(mc->mc_flags & C_EOF)) {
-
-		if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) {
-			rc = mdb_page_search(mc, NULL, MDB_PS_LAST);
-			if (rc != MDB_SUCCESS)
-				return rc;
-		}
-		mdb_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top]));
-
+	if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) {
+		rc = mdb_page_search(mc, NULL, MDB_PS_LAST);
+		if (rc != MDB_SUCCESS)
+			return rc;
 	}
+	mdb_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top]));
+
 	mc->mc_ki[mc->mc_top] = NUMKEYS(mc->mc_pg[mc->mc_top]) - 1;
 	mc->mc_flags |= C_INITIALIZED|C_EOF;
 	leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);

From 511f5880e22baddfe9fc1dee54b41a2deafc3b26 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 11 Jan 2017 10:33:28 +0000
Subject: [PATCH 290/504] Tweak cursor_next C_EOF check

Allow C_EOF flag to be stale
---
 libraries/liblmdb/mdb.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index c30a96f4c6..c84afef4dd 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -6596,15 +6596,17 @@ mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
 	MDB_node	*leaf;
 	int rc;
 
-	if ((mc->mc_flags & C_EOF) ||
-		((mc->mc_flags & C_DEL) && op == MDB_NEXT_DUP)) {
+	if ((mc->mc_flags & C_DEL && op == MDB_NEXT_DUP))
 		return MDB_NOTFOUND;
-	}
+
 	if (!(mc->mc_flags & C_INITIALIZED))
 		return mdb_cursor_first(mc, key, data);
 
 	mp = mc->mc_pg[mc->mc_top];
 
+	if ((mc->mc_flags & C_EOF) && mc->mc_ki[mc->mc_top] >= NUMKEYS(mp)-1)
+		return MDB_NOTFOUND;
+
 	if (mc->mc_db->md_flags & MDB_DUPSORT) {
 		leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]);
 		if (F_ISSET(leaf->mn_flags, F_DUPDATA)) {

From c44b29eaa84c7d62931252ec0645e24f1e1cff36 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 11 Jan 2017 11:18:57 +0000
Subject: [PATCH 291/504] ITS#8557 fix prev commit

---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index c84afef4dd..bb2068fe8d 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -6209,8 +6209,8 @@ mdb_page_search_root(MDB_cursor *mc, MDB_val *key, int flags)
 				/* if already init'd, see if we're already in right place */
 				if (mc->mc_flags & C_INITIALIZED) {
 					if (mc->mc_ki[mc->mc_top] == i) {
-						mp = mc->mc_pg[mc->mc_top];
 						mc->mc_top = mc->mc_snum++;
+						mp = mc->mc_pg[mc->mc_top];
 						goto ready;
 					}
 				}

From f8ce8a82717ddefdc912fa47c07f1bdee2a3336b Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 11 Jan 2017 11:19:18 +0000
Subject: [PATCH 292/504] More C_EOF tweaks

---
 libraries/liblmdb/mdb.c | 30 ++++++++++++++++++++++++------
 1 file changed, 24 insertions(+), 6 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index bb2068fe8d..bbea0d2b66 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -6604,8 +6604,11 @@ mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
 
 	mp = mc->mc_pg[mc->mc_top];
 
-	if ((mc->mc_flags & C_EOF) && mc->mc_ki[mc->mc_top] >= NUMKEYS(mp)-1)
-		return MDB_NOTFOUND;
+	if (mc->mc_flags & C_EOF) {
+		if (mc->mc_ki[mc->mc_top] >= NUMKEYS(mp)-1)
+			return MDB_NOTFOUND;
+		mc->mc_flags ^= C_EOF;
+	}
 
 	if (mc->mc_db->md_flags & MDB_DUPSORT) {
 		leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]);
@@ -7127,10 +7130,19 @@ mdb_cursor_get(MDB_cursor *mc, MDB_val *key, MDB_val *data,
 			rc = MDB_INCOMPATIBLE;
 			break;
 		}
-		rc = MDB_SUCCESS;
-		if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) ||
-			(mc->mc_xcursor->mx_cursor.mc_flags & C_EOF))
+		if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) {
+			rc = EINVAL;
 			break;
+		}
+		if (mc->mc_xcursor->mx_cursor.mc_flags & C_EOF) {
+			MDB_cursor *mx = &mc->mc_xcursor->mx_cursor;
+			if (mx->mc_ki[mx->mc_top] >= NUMKEYS(mx->mc_pg[mx->mc_top])-1) {
+				rc = MDB_NOTFOUND;
+				break;
+			}
+			mx->mc_flags ^= C_EOF;
+		}
+		rc = MDB_SUCCESS;
 		goto fetchm;
 	case MDB_NEXT_MULTIPLE:
 		if (data == NULL) {
@@ -8436,9 +8448,15 @@ mdb_cursor_count(MDB_cursor *mc, mdb_size_t *countp)
 	if (!(mc->mc_flags & C_INITIALIZED))
 		return EINVAL;
 
-	if (!mc->mc_snum || (mc->mc_flags & C_EOF))
+	if (!mc->mc_snum)
 		return MDB_NOTFOUND;
 
+	if (mc->mc_flags & C_EOF) {
+		if (mc->mc_ki[mc->mc_top] >= NUMKEYS(mc->mc_pg[mc->mc_top]))
+			return MDB_NOTFOUND;
+		mc->mc_flags ^= C_EOF;
+	}
+
 	leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
 	if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) {
 		*countp = 1;

From 6ac9aa666e6754f195a2e988d0c395f4d6364c28 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 11 Jan 2017 14:39:08 +0000
Subject: [PATCH 293/504] Happy New Year

---
 libraries/liblmdb/COPYRIGHT      | 2 +-
 libraries/liblmdb/intro.doc      | 2 +-
 libraries/liblmdb/lmdb.h         | 2 +-
 libraries/liblmdb/mdb.c          | 2 +-
 libraries/liblmdb/mdb_copy.1     | 4 ++--
 libraries/liblmdb/mdb_copy.c     | 2 +-
 libraries/liblmdb/mdb_dump.1     | 4 ++--
 libraries/liblmdb/mdb_dump.c     | 2 +-
 libraries/liblmdb/mdb_load.1     | 4 ++--
 libraries/liblmdb/mdb_load.c     | 2 +-
 libraries/liblmdb/mdb_stat.1     | 4 ++--
 libraries/liblmdb/mdb_stat.c     | 2 +-
 libraries/liblmdb/midl.c         | 1 +
 libraries/liblmdb/midl.h         | 1 +
 libraries/liblmdb/mtest.c        | 2 +-
 libraries/liblmdb/mtest2.c       | 2 +-
 libraries/liblmdb/mtest3.c       | 2 +-
 libraries/liblmdb/mtest4.c       | 2 +-
 libraries/liblmdb/mtest5.c       | 2 +-
 libraries/liblmdb/mtest6.c       | 2 +-
 libraries/liblmdb/sample-bdb.txt | 2 +-
 libraries/liblmdb/sample-mdb.txt | 2 +-
 22 files changed, 26 insertions(+), 24 deletions(-)

diff --git a/libraries/liblmdb/COPYRIGHT b/libraries/liblmdb/COPYRIGHT
index fe4825af57..1508a735b6 100644
--- a/libraries/liblmdb/COPYRIGHT
+++ b/libraries/liblmdb/COPYRIGHT
@@ -1,4 +1,4 @@
-Copyright 2011-2016 Howard Chu, Symas Corp.
+Copyright 2011-2017 Howard Chu, Symas Corp.
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/intro.doc b/libraries/liblmdb/intro.doc
index 9fe9c22be1..f7bd8c0d02 100644
--- a/libraries/liblmdb/intro.doc
+++ b/libraries/liblmdb/intro.doc
@@ -1,5 +1,5 @@
 /*
- * Copyright 2015-2016 Howard Chu, Symas Corp.
+ * Copyright 2015-2017 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 146bf3b5be..039210dcde 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -136,7 +136,7 @@
  *
  *	@author	Howard Chu, Symas Corporation.
  *
- *	@copyright Copyright 2011-2016 Howard Chu, Symas Corp. All rights reserved.
+ *	@copyright Copyright 2011-2017 Howard Chu, Symas Corp. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted only as authorized by the OpenLDAP
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index bbea0d2b66..3f893a8d1a 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5,7 +5,7 @@
  *	BerkeleyDB API, but much simplified.
  */
 /*
- * Copyright 2011-2016 Howard Chu, Symas Corp.
+ * Copyright 2011-2017 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mdb_copy.1 b/libraries/liblmdb/mdb_copy.1
index 4387ac3a14..3b2af35aeb 100644
--- a/libraries/liblmdb/mdb_copy.1
+++ b/libraries/liblmdb/mdb_copy.1
@@ -1,5 +1,5 @@
-.TH MDB_COPY 1 "2014/06/20" "LMDB 0.9.14"
-.\" Copyright 2012-2016 Howard Chu, Symas Corp. All Rights Reserved.
+.TH MDB_COPY 1 "2014/07/01" "LMDB 0.9.14"
+.\" Copyright 2012-2017 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_copy \- LMDB environment copy tool
diff --git a/libraries/liblmdb/mdb_copy.c b/libraries/liblmdb/mdb_copy.c
index c953201097..c6ec71c7e6 100644
--- a/libraries/liblmdb/mdb_copy.c
+++ b/libraries/liblmdb/mdb_copy.c
@@ -1,6 +1,6 @@
 /* mdb_copy.c - memory-mapped database backup tool */
 /*
- * Copyright 2012-2016 Howard Chu, Symas Corp.
+ * Copyright 2012-2017 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mdb_dump.1 b/libraries/liblmdb/mdb_dump.1
index 0b95ca175a..c4bfcb382b 100644
--- a/libraries/liblmdb/mdb_dump.1
+++ b/libraries/liblmdb/mdb_dump.1
@@ -1,5 +1,5 @@
-.TH MDB_DUMP 1 "2014/06/20" "LMDB 0.9.14"
-.\" Copyright 2014-2016 Howard Chu, Symas Corp. All Rights Reserved.
+.TH MDB_DUMP 1 "2015/09/30" "LMDB 0.9.17"
+.\" Copyright 2014-2017 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_dump \- LMDB environment export tool
diff --git a/libraries/liblmdb/mdb_dump.c b/libraries/liblmdb/mdb_dump.c
index 13084a22ec..867284d399 100644
--- a/libraries/liblmdb/mdb_dump.c
+++ b/libraries/liblmdb/mdb_dump.c
@@ -1,6 +1,6 @@
 /* mdb_dump.c - memory-mapped database dump tool */
 /*
- * Copyright 2011-2016 Howard Chu, Symas Corp.
+ * Copyright 2011-2017 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mdb_load.1 b/libraries/liblmdb/mdb_load.1
index 90ea5b9983..e25ff5a2b7 100644
--- a/libraries/liblmdb/mdb_load.1
+++ b/libraries/liblmdb/mdb_load.1
@@ -1,5 +1,5 @@
-.TH MDB_LOAD 1 "2014/06/20" "LMDB 0.9.14"
-.\" Copyright 2014-2016 Howard Chu, Symas Corp. All Rights Reserved.
+.TH MDB_LOAD 1 "2015/09/30" "LMDB 0.9.17"
+.\" Copyright 2014-2017 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_load \- LMDB environment import tool
diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c
index c421a6980f..417fad2e3a 100644
--- a/libraries/liblmdb/mdb_load.c
+++ b/libraries/liblmdb/mdb_load.c
@@ -1,6 +1,6 @@
 /* mdb_load.c - memory-mapped database load tool */
 /*
- * Copyright 2011-2016 Howard Chu, Symas Corp.
+ * Copyright 2011-2017 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mdb_stat.1 b/libraries/liblmdb/mdb_stat.1
index b5ab6a9d23..3520c92ae3 100644
--- a/libraries/liblmdb/mdb_stat.1
+++ b/libraries/liblmdb/mdb_stat.1
@@ -1,5 +1,5 @@
-.TH MDB_STAT 1 "2014/06/20" "LMDB 0.9.14"
-.\" Copyright 2012-2016 Howard Chu, Symas Corp. All Rights Reserved.
+.TH MDB_STAT 1 "2015/09/30" "LMDB 0.9.17"
+.\" Copyright 2012-2017 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_stat \- LMDB environment status tool
diff --git a/libraries/liblmdb/mdb_stat.c b/libraries/liblmdb/mdb_stat.c
index 0343a6537d..0f0fa847f3 100644
--- a/libraries/liblmdb/mdb_stat.c
+++ b/libraries/liblmdb/mdb_stat.c
@@ -1,6 +1,6 @@
 /* mdb_stat.c - memory-mapped database status tool */
 /*
- * Copyright 2011-2016 Howard Chu, Symas Corp.
+ * Copyright 2011-2017 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/midl.c b/libraries/liblmdb/midl.c
index 9748d8dba8..499df17ccb 100644
--- a/libraries/liblmdb/midl.c
+++ b/libraries/liblmdb/midl.c
@@ -4,6 +4,7 @@
 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
  *
  * Copyright 2000-2016 The OpenLDAP Foundation.
+ * Portions Copyright 2001-2017 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/midl.h b/libraries/liblmdb/midl.h
index dc532c44ff..39a87b52c2 100644
--- a/libraries/liblmdb/midl.h
+++ b/libraries/liblmdb/midl.h
@@ -12,6 +12,7 @@
 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
  *
  * Copyright 2000-2016 The OpenLDAP Foundation.
+ * Portions Copyright 2001-2017 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest.c b/libraries/liblmdb/mtest.c
index 28a33d4432..4a2eafd033 100644
--- a/libraries/liblmdb/mtest.c
+++ b/libraries/liblmdb/mtest.c
@@ -1,6 +1,6 @@
 /* mtest.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2016 Howard Chu, Symas Corp.
+ * Copyright 2011-2017 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest2.c b/libraries/liblmdb/mtest2.c
index b68aade819..9691c04fad 100644
--- a/libraries/liblmdb/mtest2.c
+++ b/libraries/liblmdb/mtest2.c
@@ -1,6 +1,6 @@
 /* mtest2.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2016 Howard Chu, Symas Corp.
+ * Copyright 2011-2017 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest3.c b/libraries/liblmdb/mtest3.c
index 73ee6e2144..4390d7ab4a 100644
--- a/libraries/liblmdb/mtest3.c
+++ b/libraries/liblmdb/mtest3.c
@@ -1,6 +1,6 @@
 /* mtest3.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2016 Howard Chu, Symas Corp.
+ * Copyright 2011-2017 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest4.c b/libraries/liblmdb/mtest4.c
index 25666fffd3..9f7884a93f 100644
--- a/libraries/liblmdb/mtest4.c
+++ b/libraries/liblmdb/mtest4.c
@@ -1,6 +1,6 @@
 /* mtest4.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2016 Howard Chu, Symas Corp.
+ * Copyright 2011-2017 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest5.c b/libraries/liblmdb/mtest5.c
index d3d7b6286d..4d34247e60 100644
--- a/libraries/liblmdb/mtest5.c
+++ b/libraries/liblmdb/mtest5.c
@@ -1,6 +1,6 @@
 /* mtest5.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2016 Howard Chu, Symas Corp.
+ * Copyright 2011-2017 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest6.c b/libraries/liblmdb/mtest6.c
index 1b47db83f5..26904e88ba 100644
--- a/libraries/liblmdb/mtest6.c
+++ b/libraries/liblmdb/mtest6.c
@@ -1,6 +1,6 @@
 /* mtest6.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2016 Howard Chu, Symas Corp.
+ * Copyright 2011-2017 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/sample-bdb.txt b/libraries/liblmdb/sample-bdb.txt
index 11aff13f2e..4de26d5585 100644
--- a/libraries/liblmdb/sample-bdb.txt
+++ b/libraries/liblmdb/sample-bdb.txt
@@ -3,7 +3,7 @@
  * Do a line-by-line comparison of this and sample-mdb.txt
  */
 /*
- * Copyright 2012-2016 Howard Chu, Symas Corp.
+ * Copyright 2012-2017 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/sample-mdb.txt b/libraries/liblmdb/sample-mdb.txt
index 5d3373736d..6e3622b866 100644
--- a/libraries/liblmdb/sample-mdb.txt
+++ b/libraries/liblmdb/sample-mdb.txt
@@ -3,7 +3,7 @@
  * Do a line-by-line comparison of this and sample-bdb.txt
  */
 /*
- * Copyright 2012-2016 Howard Chu, Symas Corp.
+ * Copyright 2012-2017 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without

From 5eae7aad7e4db9f354fa8a5a52eab9646e07a028 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 12 Jan 2017 13:35:31 +0000
Subject: [PATCH 294/504] Fix f8ce8a82717ddefdc912fa47c07f1bdee2a3336b

GET_MULTIPLE was broken
---
 libraries/liblmdb/mdb.c | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 3f893a8d1a..649ff01035 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7130,10 +7130,9 @@ mdb_cursor_get(MDB_cursor *mc, MDB_val *key, MDB_val *data,
 			rc = MDB_INCOMPATIBLE;
 			break;
 		}
-		if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) {
-			rc = EINVAL;
+		rc = MDB_SUCCESS;
+		if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED))
 			break;
-		}
 		if (mc->mc_xcursor->mx_cursor.mc_flags & C_EOF) {
 			MDB_cursor *mx = &mc->mc_xcursor->mx_cursor;
 			if (mx->mc_ki[mx->mc_top] >= NUMKEYS(mx->mc_pg[mx->mc_top])-1) {
@@ -7142,7 +7141,6 @@ mdb_cursor_get(MDB_cursor *mc, MDB_val *key, MDB_val *data,
 			}
 			mx->mc_flags ^= C_EOF;
 		}
-		rc = MDB_SUCCESS;
 		goto fetchm;
 	case MDB_NEXT_MULTIPLE:
 		if (data == NULL) {

From 882e27c1b4f40270e712c251d5454c7e93424f58 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sat, 14 Jan 2017 19:22:34 +0000
Subject: [PATCH 295/504] Further fix f8ce8a82717ddefdc912fa47c07f1bdee2a3336b

Fully revert the change to GET_MULTIPLE
---
 libraries/liblmdb/mdb.c | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 649ff01035..82bac14bbb 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7131,16 +7131,9 @@ mdb_cursor_get(MDB_cursor *mc, MDB_val *key, MDB_val *data,
 			break;
 		}
 		rc = MDB_SUCCESS;
-		if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED))
+		if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) ||
+			(mc->mc_xcursor->mx_cursor.mc_flags & C_EOF))
 			break;
-		if (mc->mc_xcursor->mx_cursor.mc_flags & C_EOF) {
-			MDB_cursor *mx = &mc->mc_xcursor->mx_cursor;
-			if (mx->mc_ki[mx->mc_top] >= NUMKEYS(mx->mc_pg[mx->mc_top])-1) {
-				rc = MDB_NOTFOUND;
-				break;
-			}
-			mx->mc_flags ^= C_EOF;
-		}
 		goto fetchm;
 	case MDB_NEXT_MULTIPLE:
 		if (data == NULL) {

From 1db9f32ae23330de44e86419991d64202ce9ac92 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 31 Jan 2017 10:41:52 +0000
Subject: [PATCH 296/504] Workaround VL32 cursor refcounting miscount

Don't try to deref cursor page if txn's pagelist is empty
---
 libraries/liblmdb/mdb.c | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 82bac14bbb..ec2488a89a 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2006,13 +2006,15 @@ static void
 mdb_cursor_unref(MDB_cursor *mc)
 {
 	int i;
-	if (!mc->mc_snum || !mc->mc_pg[0] || IS_SUBP(mc->mc_pg[0]))
-		return;
-	for (i=0; i<mc->mc_snum; i++)
-		mdb_page_unref(mc->mc_txn, mc->mc_pg[i]);
-	if (mc->mc_ovpg) {
-		mdb_page_unref(mc->mc_txn, mc->mc_ovpg);
-		mc->mc_ovpg = 0;
+	if (mc->mc_txn->mt_rpages[0].mid) {
+		if (!mc->mc_snum || !mc->mc_pg[0] || IS_SUBP(mc->mc_pg[0]))
+			return;
+		for (i=0; i<mc->mc_snum; i++)
+			mdb_page_unref(mc->mc_txn, mc->mc_pg[i]);
+		if (mc->mc_ovpg) {
+			mdb_page_unref(mc->mc_txn, mc->mc_ovpg);
+			mc->mc_ovpg = 0;
+		}
 	}
 	mc->mc_snum = mc->mc_top = 0;
 	mc->mc_pg[0] = NULL;

From e36517dbbe157207cde3289776b76ae6d7bd1877 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 6 Feb 2017 15:09:26 +0000
Subject: [PATCH 297/504] ITS#8582 keep mutex at end of struct

since it's variable size on Linux/glibc
---
 libraries/liblmdb/mdb.c | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index ec2488a89a..ac381d9e2b 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -809,6 +809,16 @@ typedef struct MDB_txbody {
 	uint32_t	mtb_magic;
 		/** Format of this lock file. Must be set to #MDB_LOCK_FORMAT. */
 	uint32_t	mtb_format;
+		/**	The ID of the last transaction committed to the database.
+		 *	This is recorded here only for convenience; the value can always
+		 *	be determined by reading the main database meta pages.
+		 */
+	volatile txnid_t		mtb_txnid;
+		/** The number of slots that have been used in the reader table.
+		 *	This always records the maximum count, it is not decremented
+		 *	when readers release their slots.
+		 */
+	volatile unsigned	mtb_numreaders;
 #if defined(_WIN32) || defined(MDB_USE_POSIX_SEM)
 	char	mtb_rmname[MNAME_LEN];
 #elif defined(MDB_USE_SYSV_SEM)
@@ -820,16 +830,6 @@ typedef struct MDB_txbody {
 		 */
 	mdb_mutex_t	mtb_rmutex;
 #endif
-		/**	The ID of the last transaction committed to the database.
-		 *	This is recorded here only for convenience; the value can always
-		 *	be determined by reading the main database meta pages.
-		 */
-	volatile txnid_t		mtb_txnid;
-		/** The number of slots that have been used in the reader table.
-		 *	This always records the maximum count, it is not decremented
-		 *	when readers release their slots.
-		 */
-	volatile unsigned	mtb_numreaders;
 } MDB_txbody;
 
 	/** The actual reader table definition. */

From 58ba039b8f30733ad9456cb3a31f9c459e6e0ffe Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 4 Mar 2017 13:03:15 +0100
Subject: [PATCH 298/504] ITS#8582 MDB_LOCK_VERSION = 2 due to format change

---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index ac381d9e2b..21f50f2a88 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -620,7 +620,7 @@ static txnid_t mdb_debug_start;
 	/**	The version number for a database's datafile format. */
 #define MDB_DATA_VERSION	 ((MDB_DEVEL) ? 999 : 1)
 	/**	The version number for a database's lockfile format. */
-#define MDB_LOCK_VERSION	 ((MDB_DEVEL) ? 999 : 1)
+#define MDB_LOCK_VERSION	 ((MDB_DEVEL) ? 999 : 2)
 
 	/**	@brief The max size of a key we can write, or 0 for computed max.
 	 *

From 68eda68f0b99bd766d1fd52658e088640b67b39c Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sun, 12 Mar 2017 16:55:14 +0100
Subject: [PATCH 299/504] Store lock ID instead of pathname in lockfile

This limits the namespace which the user can meddle with for
POSIX semaphores and Windows mutexes.  Their names change a
bit, they no longer have fixed lengths.
---
 libraries/liblmdb/mdb.c | 101 ++++++++++++++++++++++++----------------
 1 file changed, 61 insertions(+), 40 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 21f50f2a88..d28f42ae93 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -465,14 +465,21 @@ typedef pthread_mutex_t *mdb_mutexref_t;
 #define	Yu	MDB_PRIy(u)	/**< printf format for #mdb_size_t */
 #define	Yd	MDB_PRIy(d)	/**< printf format for 'signed #mdb_size_t' */
 
-#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM)
-#define MNAME_LEN	32
-#elif defined(MDB_USE_SYSV_SEM)
+#ifdef MDB_USE_SYSV_SEM
 #define MNAME_LEN	(sizeof(int))
 #else
 #define MNAME_LEN	(sizeof(pthread_mutex_t))
 #endif
 
+/** Initial part of #MDB_env.me_mutexname[].
+ *	Changes to this code must be reflected in #MDB_LOCK_FORMAT.
+ */
+#ifdef _WIN32
+#define MUTEXNAME_PREFIX		"Global\\MDB"
+#elif defined MDB_USE_POSIX_SEM
+#define MUTEXNAME_PREFIX		"/MDB"
+#endif
+
 #ifdef MDB_USE_SYSV_SEM
 #define SYSV_SEM_FLAG	1		/**< SysV sems in lockfile format */
 #else
@@ -692,6 +699,8 @@ static txnid_t mdb_debug_start;
 	 */
 typedef uint16_t	 indx_t;
 
+typedef unsigned long long	mdb_hash_t;
+
 	/**	Default size of memory map.
 	 *	This is certainly too small for any actual applications. Apps should always set
 	 *	the size explicitly using #mdb_env_set_mapsize().
@@ -820,7 +829,8 @@ typedef struct MDB_txbody {
 		 */
 	volatile unsigned	mtb_numreaders;
 #if defined(_WIN32) || defined(MDB_USE_POSIX_SEM)
-	char	mtb_rmname[MNAME_LEN];
+		/** Binary form of names of the reader/writer locks */
+	mdb_hash_t			mtb_mutexid;
 #elif defined(MDB_USE_SYSV_SEM)
 	int 	mtb_semid;
 	int		mtb_rlocked;
@@ -842,17 +852,16 @@ typedef struct MDB_txninfo {
 #define mti_rmname	mt1.mtb.mtb_rmname
 #define mti_txnid	mt1.mtb.mtb_txnid
 #define mti_numreaders	mt1.mtb.mtb_numreaders
+#define mti_mutexid	mt1.mtb.mtb_mutexid
 #ifdef MDB_USE_SYSV_SEM
 #define	mti_semid	mt1.mtb.mtb_semid
 #define	mti_rlocked	mt1.mtb.mtb_rlocked
 #endif
 		char pad[(sizeof(MDB_txbody)+CACHELINE-1) & ~(CACHELINE-1)];
 	} mt1;
+#if !(defined(_WIN32) || defined(MDB_USE_POSIX_SEM))
 	union {
-#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM)
-		char mt2_wmname[MNAME_LEN];
-#define	mti_wmname	mt2.mt2_wmname
-#elif defined MDB_USE_SYSV_SEM
+#ifdef MDB_USE_SYSV_SEM
 		int mt2_wlocked;
 #define mti_wlocked	mt2.mt2_wlocked
 #else
@@ -861,6 +870,7 @@ typedef struct MDB_txninfo {
 #endif
 		char pad[(MNAME_LEN+CACHELINE-1) & ~(CACHELINE-1)];
 	} mt2;
+#endif
 	MDB_reader	mti_readers[1];
 } MDB_txninfo;
 
@@ -1447,6 +1457,10 @@ struct MDB_env {
 #else
 	mdb_mutex_t	me_rmutex;
 	mdb_mutex_t	me_wmutex;
+# if defined(_WIN32) || defined(MDB_USE_POSIX_SEM)
+	/** Half-initialized name of mutexes, to be completed by #MUTEXNAME() */
+	char		me_mutexname[sizeof(MUTEXNAME_PREFIX) + 11];
+# endif
 #endif
 #ifdef MDB_VL32
 	MDB_ID3L	me_rpages;	/**< like #mt_rpages, but global to env */
@@ -4951,7 +4965,6 @@ mdb_env_excl_lock(MDB_env *env, int *excl)
  * Share and Enjoy!	:-)
  */
 
-typedef unsigned long long	mdb_hash_t;
 #define MDB_HASH_INIT ((mdb_hash_t)0xcbf29ce484222325ULL)
 
 /** perform a 64 bit Fowler/Noll/Vo FNV-1a hash on a buffer
@@ -4993,25 +5006,33 @@ mdb_hash_val(MDB_val *val, mdb_hash_t hval)
 static const char mdb_a85[]= "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~";
 
 static void ESECT
-mdb_pack85(unsigned long l, char *out)
+mdb_pack85(unsigned long long l, char *out)
 {
 	int i;
 
-	for (i=0; i<5; i++) {
+	for (i=0; i<10 && l; i++) {
 		*out++ = mdb_a85[l % 85];
 		l /= 85;
 	}
+	*out = '\0';
 }
 
+/** Init #MDB_env.me_mutexname[] except the char which #MUTEXNAME() will set.
+ *	Changes to this code must be reflected in #MDB_LOCK_FORMAT.
+ */
 static void ESECT
-mdb_hash_enc(MDB_val *val, char *encbuf)
+mdb_env_mname_init(MDB_env *env)
 {
-	mdb_hash_t h = mdb_hash_val(val, MDB_HASH_INIT);
-
-	mdb_pack85(h, encbuf);
-	mdb_pack85(h>>32, encbuf+5);
-	encbuf[10] = '\0';
+	char *nm = env->me_mutexname;
+	strcpy(nm, MUTEXNAME_PREFIX);
+	mdb_pack85(env->me_txns->mti_mutexid, nm + sizeof(MUTEXNAME_PREFIX));
 }
+
+/** Return env->me_mutexname after filling in ch ('r'/'w') for convenience */
+#define MUTEXNAME(env, ch) ( \
+		(void) ((env)->me_mutexname[sizeof(MUTEXNAME_PREFIX)-1] = (ch)), \
+		(env)->me_mutexname)
+
 #endif
 
 /** Open and/or initialize the lock region for the environment.
@@ -5110,7 +5131,6 @@ mdb_env_setup_locks(MDB_env *env, MDB_name *fname, int mode, int *excl)
 			DWORD nlow;
 		} idbuf;
 		MDB_val val;
-		char encbuf[11];
 
 		if (!mdb_sec_inited) {
 			InitializeSecurityDescriptor(&mdb_null_sd,
@@ -5127,12 +5147,11 @@ mdb_env_setup_locks(MDB_env *env, MDB_name *fname, int mode, int *excl)
 		idbuf.nlow   = stbuf.nFileIndexLow;
 		val.mv_data = &idbuf;
 		val.mv_size = sizeof(idbuf);
-		mdb_hash_enc(&val, encbuf);
-		sprintf(env->me_txns->mti_rmname, "Global\\MDBr%s", encbuf);
-		sprintf(env->me_txns->mti_wmname, "Global\\MDBw%s", encbuf);
-		env->me_rmutex = CreateMutexA(&mdb_all_sa, FALSE, env->me_txns->mti_rmname);
+		env->me_txns->mti_mutexid = mdb_hash_val(&val, MDB_HASH_INIT);
+		mdb_env_mname_init(env);
+		env->me_rmutex = CreateMutexA(&mdb_all_sa, FALSE, MUTEXNAME(env, 'r'));
 		if (!env->me_rmutex) goto fail_errno;
-		env->me_wmutex = CreateMutexA(&mdb_all_sa, FALSE, env->me_txns->mti_wmname);
+		env->me_wmutex = CreateMutexA(&mdb_all_sa, FALSE, MUTEXNAME(env, 'w'));
 		if (!env->me_wmutex) goto fail_errno;
 #elif defined(MDB_USE_POSIX_SEM)
 		struct stat stbuf;
@@ -5141,7 +5160,6 @@ mdb_env_setup_locks(MDB_env *env, MDB_name *fname, int mode, int *excl)
 			ino_t ino;
 		} idbuf;
 		MDB_val val;
-		char encbuf[11];
 
 #if defined(__NetBSD__)
 #define	MDB_SHORT_SEMNAMES	1	/* limited to 14 chars */
@@ -5151,22 +5169,23 @@ mdb_env_setup_locks(MDB_env *env, MDB_name *fname, int mode, int *excl)
 		idbuf.ino = stbuf.st_ino;
 		val.mv_data = &idbuf;
 		val.mv_size = sizeof(idbuf);
-		mdb_hash_enc(&val, encbuf);
+		env->me_txns->mti_mutexid = mdb_hash_val(&val, MDB_HASH_INIT)
 #ifdef MDB_SHORT_SEMNAMES
-		encbuf[9] = '\0';	/* drop name from 15 chars to 14 chars */
+			/* Max 9 base85-digits.  We truncate here instead of in
+			 * mdb_env_mname_init() to keep the latter portable.
+			 */
+			% ((mdb_hash_t)85*85*85*85*85*85*85*85*85)
 #endif
-		sprintf(env->me_txns->mti_rmname, "/MDBr%s", encbuf);
-		sprintf(env->me_txns->mti_wmname, "/MDBw%s", encbuf);
+			;
+		mdb_env_mname_init(env);
 		/* Clean up after a previous run, if needed:  Try to
 		 * remove both semaphores before doing anything else.
 		 */
-		sem_unlink(env->me_txns->mti_rmname);
-		sem_unlink(env->me_txns->mti_wmname);
-		env->me_rmutex = sem_open(env->me_txns->mti_rmname,
-			O_CREAT|O_EXCL, mode, 1);
+		sem_unlink(MUTEXNAME(env, 'r'));
+		sem_unlink(MUTEXNAME(env, 'w'));
+		env->me_rmutex = sem_open(MUTEXNAME(env, 'r'), O_CREAT|O_EXCL, mode, 1);
 		if (env->me_rmutex == SEM_FAILED) goto fail_errno;
-		env->me_wmutex = sem_open(env->me_txns->mti_wmname,
-			O_CREAT|O_EXCL, mode, 1);
+		env->me_wmutex = sem_open(MUTEXNAME(env, 'w'), O_CREAT|O_EXCL, mode, 1);
 		if (env->me_wmutex == SEM_FAILED) goto fail_errno;
 #elif defined(MDB_USE_SYSV_SEM)
 		unsigned short vals[2] = {1, 1};
@@ -5230,14 +5249,16 @@ mdb_env_setup_locks(MDB_env *env, MDB_name *fname, int mode, int *excl)
 			goto fail;
 		}
 #ifdef _WIN32
-		env->me_rmutex = OpenMutexA(SYNCHRONIZE, FALSE, env->me_txns->mti_rmname);
+		mdb_env_mname_init(env);
+		env->me_rmutex = OpenMutexA(SYNCHRONIZE, FALSE, MUTEXNAME(env, 'r'));
 		if (!env->me_rmutex) goto fail_errno;
-		env->me_wmutex = OpenMutexA(SYNCHRONIZE, FALSE, env->me_txns->mti_wmname);
+		env->me_wmutex = OpenMutexA(SYNCHRONIZE, FALSE, MUTEXNAME(env, 'w'));
 		if (!env->me_wmutex) goto fail_errno;
 #elif defined(MDB_USE_POSIX_SEM)
-		env->me_rmutex = sem_open(env->me_txns->mti_rmname, 0);
+		mdb_env_mname_init(env);
+		env->me_rmutex = sem_open(MUTEXNAME(env, 'r'), 0);
 		if (env->me_rmutex == SEM_FAILED) goto fail_errno;
-		env->me_wmutex = sem_open(env->me_txns->mti_wmname, 0);
+		env->me_wmutex = sem_open(MUTEXNAME(env, 'w'), 0);
 		if (env->me_wmutex == SEM_FAILED) goto fail_errno;
 #elif defined(MDB_USE_SYSV_SEM)
 		semid = env->me_txns->mti_semid;
@@ -5517,8 +5538,8 @@ mdb_env_close0(MDB_env *env, int excl)
 			if (excl == 0)
 				mdb_env_excl_lock(env, &excl);
 			if (excl > 0) {
-				sem_unlink(env->me_txns->mti_rmname);
-				sem_unlink(env->me_txns->mti_wmname);
+				sem_unlink(MUTEXNAME(env, 'r'));
+				sem_unlink(MUTEXNAME(env, 'w'));
 			}
 		}
 #elif defined(MDB_USE_SYSV_SEM)

From 172d825155a7ba595527181277983ba4022c2115 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sun, 12 Mar 2017 16:58:51 +0100
Subject: [PATCH 300/504] Simplify mdb_hash_val() -> mdb_hash()

Simpler usage since it's only called once, rename to match new usage,
and drop 3 loop pessimizations which were optimizations 20 years ago.
---
 libraries/liblmdb/mdb.c | 32 ++++++++------------------------
 1 file changed, 8 insertions(+), 24 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index d28f42ae93..e2b48589e0 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -4965,31 +4965,21 @@ mdb_env_excl_lock(MDB_env *env, int *excl)
  * Share and Enjoy!	:-)
  */
 
-#define MDB_HASH_INIT ((mdb_hash_t)0xcbf29ce484222325ULL)
-
 /** perform a 64 bit Fowler/Noll/Vo FNV-1a hash on a buffer
  * @param[in] val	value to hash
- * @param[in] hval	initial value for hash
+ * @param[in] len	length of value
  * @return 64 bit hash
- *
- * NOTE: To use the recommended 64 bit FNV-1a hash, use MDB_HASH_INIT as the
- * 	 hval arg on the first call.
  */
 static mdb_hash_t
-mdb_hash_val(MDB_val *val, mdb_hash_t hval)
+mdb_hash(const void *val, size_t len)
 {
-	unsigned char *s = (unsigned char *)val->mv_data;	/* unsigned string */
-	unsigned char *end = s + val->mv_size;
+	const unsigned char *s = (const unsigned char *) val, *end = s + len;
+	mdb_hash_t hval = 0xcbf29ce484222325ULL;
 	/*
-	 * FNV-1a hash each octet of the string
+	 * FNV-1a hash each octet of the buffer
 	 */
 	while (s < end) {
-		/* xor the bottom with the current octet */
-		hval ^= (mdb_hash_t)*s++;
-
-		/* multiply by the 64 bit FNV magic prime mod 2^64 */
-		hval += (hval << 1) + (hval << 4) + (hval << 5) +
-			(hval << 7) + (hval << 8) + (hval << 40);
+		hval = (hval ^ *s++) * 0x100000001b3ULL;
 	}
 	/* return our new hash value */
 	return hval;
@@ -5130,7 +5120,6 @@ mdb_env_setup_locks(MDB_env *env, MDB_name *fname, int mode, int *excl)
 			DWORD nhigh;
 			DWORD nlow;
 		} idbuf;
-		MDB_val val;
 
 		if (!mdb_sec_inited) {
 			InitializeSecurityDescriptor(&mdb_null_sd,
@@ -5145,9 +5134,7 @@ mdb_env_setup_locks(MDB_env *env, MDB_name *fname, int mode, int *excl)
 		idbuf.volume = stbuf.dwVolumeSerialNumber;
 		idbuf.nhigh  = stbuf.nFileIndexHigh;
 		idbuf.nlow   = stbuf.nFileIndexLow;
-		val.mv_data = &idbuf;
-		val.mv_size = sizeof(idbuf);
-		env->me_txns->mti_mutexid = mdb_hash_val(&val, MDB_HASH_INIT);
+		env->me_txns->mti_mutexid = mdb_hash(&idbuf, sizeof(idbuf));
 		mdb_env_mname_init(env);
 		env->me_rmutex = CreateMutexA(&mdb_all_sa, FALSE, MUTEXNAME(env, 'r'));
 		if (!env->me_rmutex) goto fail_errno;
@@ -5159,7 +5146,6 @@ mdb_env_setup_locks(MDB_env *env, MDB_name *fname, int mode, int *excl)
 			dev_t dev;
 			ino_t ino;
 		} idbuf;
-		MDB_val val;
 
 #if defined(__NetBSD__)
 #define	MDB_SHORT_SEMNAMES	1	/* limited to 14 chars */
@@ -5167,9 +5153,7 @@ mdb_env_setup_locks(MDB_env *env, MDB_name *fname, int mode, int *excl)
 		if (fstat(env->me_lfd, &stbuf)) goto fail_errno;
 		idbuf.dev = stbuf.st_dev;
 		idbuf.ino = stbuf.st_ino;
-		val.mv_data = &idbuf;
-		val.mv_size = sizeof(idbuf);
-		env->me_txns->mti_mutexid = mdb_hash_val(&val, MDB_HASH_INIT)
+		env->me_txns->mti_mutexid = mdb_hash(&idbuf, sizeof(idbuf))
 #ifdef MDB_SHORT_SEMNAMES
 			/* Max 9 base85-digits.  We truncate here instead of in
 			 * mdb_env_mname_init() to keep the latter portable.

From 52c0df1d5052296200d1db607ba4f6bd6eb35f4d Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sun, 12 Mar 2017 16:59:23 +0100
Subject: [PATCH 301/504] Clear any struct padding in idbuf

---
 libraries/liblmdb/mdb.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index e2b48589e0..59e3d938b1 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5151,6 +5151,7 @@ mdb_env_setup_locks(MDB_env *env, MDB_name *fname, int mode, int *excl)
 #define	MDB_SHORT_SEMNAMES	1	/* limited to 14 chars */
 #endif
 		if (fstat(env->me_lfd, &stbuf)) goto fail_errno;
+		memset(&idbuf, 0, sizeof(idbuf));
 		idbuf.dev = stbuf.st_dev;
 		idbuf.ino = stbuf.st_ino;
 		env->me_txns->mti_mutexid = mdb_hash(&idbuf, sizeof(idbuf))

From b5e5fcc31dd6719a618cd825a0a9c4ecef5d045b Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sun, 12 Mar 2017 20:08:53 +0100
Subject: [PATCH 302/504] ITS#8582 Fill in MDB_LOCK_FORMAT

Attempt to stop liblmdb variants compiled with conflicting options
from using the lockfile at the same time and thus breaking it.
---
 libraries/liblmdb/mdb.c | 77 ++++++++++++++++++++++++++++++++++-------
 1 file changed, 65 insertions(+), 12 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 59e3d938b1..4a364f7fdb 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -480,12 +480,6 @@ typedef pthread_mutex_t *mdb_mutexref_t;
 #define MUTEXNAME_PREFIX		"/MDB"
 #endif
 
-#ifdef MDB_USE_SYSV_SEM
-#define SYSV_SEM_FLAG	1		/**< SysV sems in lockfile format */
-#else
-#define SYSV_SEM_FLAG	0
-#endif
-
 /** @} */
 
 #ifdef MDB_ROBUST_SUPPORTED
@@ -628,6 +622,10 @@ static txnid_t mdb_debug_start;
 #define MDB_DATA_VERSION	 ((MDB_DEVEL) ? 999 : 1)
 	/**	The version number for a database's lockfile format. */
 #define MDB_LOCK_VERSION	 ((MDB_DEVEL) ? 999 : 2)
+	/** Number of bits representing #MDB_LOCK_VERSION in #MDB_LOCK_FORMAT.
+	 *	The remaining bits must leave room for #MDB_lock_desc.
+	 */
+#define MDB_LOCK_VERSION_BITS 12
 
 	/**	@brief The max size of a key we can write, or 0 for computed max.
 	 *
@@ -693,6 +691,19 @@ static txnid_t mdb_debug_start;
 	/** Round \b n up to an even number. */
 #define EVEN(n)		(((n) + 1U) & -2) /* sign-extending -2 to match n+1U */
 
+	/** Least significant 1-bit of \b n.  n must be of an unsigned type. */
+#define LOW_BIT(n)		((n) & (-(n)))
+
+	/** (log2(\b p2) % \b n), for p2 = power of 2 and 0 < n < 8. */
+#define LOG2_MOD(p2, n)	(7 - 86 / ((p2) % ((1U<<(n))-1) + 11))
+	/* Explanation: Let p2 = 2**(n*y + x), x<n and M = (1U<<n)-1. Now p2 =
+	 * (M+1)**y * 2**x = 2**x (mod M). Finally "/" "happens" to return 7-x.
+	 */
+
+	/** Should be alignment of \b type. Ensure it is a power of 2. */
+#define ALIGNOF2(type) \
+	LOW_BIT(offsetof(struct { char ch_; type align_; }, align_))
+
 	/**	Used for offsets within a single page.
 	 *	Since memory pages are typically 4 or 8KB in size, 12-13 bits,
 	 *	this is plenty.
@@ -849,7 +860,6 @@ typedef struct MDB_txninfo {
 #define mti_magic	mt1.mtb.mtb_magic
 #define mti_format	mt1.mtb.mtb_format
 #define mti_rmutex	mt1.mtb.mtb_rmutex
-#define mti_rmname	mt1.mtb.mtb_rmname
 #define mti_txnid	mt1.mtb.mtb_txnid
 #define mti_numreaders	mt1.mtb.mtb_numreaders
 #define mti_mutexid	mt1.mtb.mtb_mutexid
@@ -876,11 +886,54 @@ typedef struct MDB_txninfo {
 
 	/** Lockfile format signature: version, features and field layout */
 #define MDB_LOCK_FORMAT \
-	((uint32_t) \
-	 ((MDB_LOCK_VERSION) \
-	  /* Flags which describe functionality */ \
-	  + (SYSV_SEM_FLAG << 18) \
-	  + (((MDB_PIDLOCK) != 0) << 16)))
+	((uint32_t)         \
+	 (((MDB_LOCK_VERSION) % (1U << MDB_LOCK_VERSION_BITS)) \
+	  + MDB_lock_desc     * (1U << MDB_LOCK_VERSION_BITS)))
+
+	/** Lock type and layout. Values 0-119. _WIN32 implies #MDB_PIDLOCK.
+	 *	Some low values are reserved for future tweaks.
+	 */
+#ifdef _WIN32
+# define MDB_LOCK_TYPE	(0 + ALIGNOF2(mdb_hash_t)/8 % 2)
+#elif defined MDB_USE_POSIX_SEM
+# define MDB_LOCK_TYPE	(4 + ALIGNOF2(mdb_hash_t)/8 % 2)
+#elif defined MDB_USE_SYSV_SEM
+# define MDB_LOCK_TYPE	(8)
+#elif defined MDB_USE_POSIX_MUTEX
+/* We do not know the inside of a POSIX mutex and how to check if mutexes
+ * used by two executables are compatible. Just check alignment and size.
+ */
+# define MDB_LOCK_TYPE	(10 + \
+		LOG2_MOD(ALIGNOF2(pthread_mutex_t), 5) + \
+		sizeof(pthread_mutex_t) / 4U % 22 * 5)
+#endif
+
+enum {
+	/** Magic number for lockfile layout and features.
+	 *
+	 *  This *attempts* to stop liblmdb variants compiled with conflicting
+	 *	options from using the lockfile at the same time and thus breaking
+	 *	it.  It describes locking types, and sizes and sometimes alignment
+	 *	of the various lockfile items.
+	 *
+	 *	The detected ranges are mostly guesswork, or based simply on how
+	 *	big they could be without using more bits.  So we can tweak them
+	 *	in good conscience when updating #MDB_LOCK_VERSION.
+	 */
+	MDB_lock_desc =
+	/* Default CACHELINE=64 vs. other values (have seen mention of 32-256) */
+	(CACHELINE==64 ? 0 : 1 + LOG2_MOD(CACHELINE >> (CACHELINE>64), 5))
+	+ 6  * (sizeof(MDB_PID_T)/4 % 3)    /* legacy(2) to word(4/8)? */
+	+ 18 * (sizeof(pthread_t)/4 % 5)    /* can be struct{id, active data} */
+	+ 90 * (sizeof(MDB_txbody) / CACHELINE % 3)
+	+ 270 * (MDB_LOCK_TYPE % 120)
+	/* The above is < 270*120 < 2**15 */
+	+ ((sizeof(txnid_t) == 8) << 15)    /* 32bit/64bit */
+	+ ((sizeof(MDB_reader) > CACHELINE) << 16)
+	/* Not really needed - implied by MDB_LOCK_TYPE != (_WIN32 locking) */
+	+ (((MDB_PIDLOCK) != 0)   << 17)
+	/* 18 bits total: Must be <= (32 - MDB_LOCK_VERSION_BITS). */
+};
 /** @} */
 
 /** Common header for all page types. The page type depends on #mp_flags.

From 4d2154397afd90ca519bfa102b2aad515159bd50 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 23 Mar 2017 20:37:24 +0000
Subject: [PATCH 303/504] ITS#8622 fix xcursor after cursor_del

Re-fix 6b1df0e4c7fadd21d1233d7157229b2d89ccaa04 from ITS#8406
---
 libraries/liblmdb/mdb.c | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 4a364f7fdb..970e1ef28e 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -9302,14 +9302,17 @@ mdb_cursor_del0(MDB_cursor *mc)
 					}
 					if (mc->mc_db->md_flags & MDB_DUPSORT) {
 						MDB_node *node = NODEPTR(m3->mc_pg[m3->mc_top], m3->mc_ki[m3->mc_top]);
-						/* If this node is a fake page, it needs to be reinited
-						 * because its data has moved. But just reset mc_pg[0]
-						 * if the xcursor is already live.
+						/* If this node has dupdata, it may need to be reinited
+						 * because its data has moved.
+						 * If the xcursor was not initd it must be reinited.
+						 * Else if node points to a subDB, nothing is needed.
+						 * Else (xcursor was initd, not a subDB) needs mc_pg[0] reset.
 						 */
-						if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) {
-							if (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)
-								m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node);
-							else
+						if (node->mn_flags & F_DUPDATA) {
+							if (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) {
+								if (!(node->mn_flags & F_SUBDATA))
+									m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node);
+							} else
 								mdb_xcursor_init1(m3, node);
 						}
 					}

From 348885413ef0688f75b737dd3c75378b93e7b66d Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 7 Jun 2016 23:38:47 +0100
Subject: [PATCH 304/504] MDB_VL32

Use the same size dirty list for both 64 and 32 bit.
---
 libraries/liblmdb/midl.h | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/libraries/liblmdb/midl.h b/libraries/liblmdb/midl.h
index 39a87b52c2..48591dd772 100644
--- a/libraries/liblmdb/midl.h
+++ b/libraries/liblmdb/midl.h
@@ -56,11 +56,7 @@ typedef MDB_ID *MDB_IDL;
 /* IDL sizes - likely should be even bigger
  *   limiting factors: sizeof(ID), thread stack size
  */
-#ifdef MDB_VL32
-#define	MDB_IDL_LOGN	14	/* DB_SIZE is 2^14, UM_SIZE is 2^15 */
-#else
 #define	MDB_IDL_LOGN	16	/* DB_SIZE is 2^16, UM_SIZE is 2^17 */
-#endif
 #define MDB_IDL_DB_SIZE		(1<<MDB_IDL_LOGN)
 #define MDB_IDL_UM_SIZE		(1<<(MDB_IDL_LOGN+1))
 

From 775be5e50f73ef0135cfb3a803d1abba22532cc0 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 12 Apr 2017 23:55:29 +0100
Subject: [PATCH 305/504] Fix Android recognition

The official macro is __ANDROID__; ANDROID may or may not be defined.
---
 libraries/liblmdb/mdb.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 970e1ef28e..a64f60a140 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -138,7 +138,7 @@ typedef SSIZE_T	ssize_t;
 #include <unistd.h>
 #endif
 
-#if defined(__sun) || defined(ANDROID)
+#if defined(__sun) || defined(__ANDROID__)
 /* Most platforms have posix_memalign, older may only have memalign */
 #define HAVE_MEMALIGN	1
 #include <malloc.h>
@@ -154,7 +154,7 @@ typedef SSIZE_T	ssize_t;
 # define MDB_USE_SYSV_SEM	1
 # endif
 # define MDB_FDATASYNC		fsync
-#elif defined(ANDROID)
+#elif defined(__ANDROID__)
 # define MDB_FDATASYNC		fsync
 #endif
 
@@ -300,7 +300,7 @@ union semun {
  */
 #ifndef MDB_USE_ROBUST
 /* Android currently lacks Robust Mutex support. So does glibc < 2.4. */
-# if defined(MDB_USE_POSIX_MUTEX) && (defined(ANDROID) || \
+# if defined(MDB_USE_POSIX_MUTEX) && (defined(__ANDROID__) || \
 	(defined(__GLIBC__) && GLIBC_VER < 0x020004))
 #  define MDB_USE_ROBUST	0
 # else

From ac047d1eff8a7cfcd7933f013aa47609f82ef200 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sat, 27 May 2017 15:37:14 +0100
Subject: [PATCH 306/504] Add new MDB_RPAGE_CACHE def

Separate 32/64 dependency from rpage buffer mechanism
---
 libraries/liblmdb/lmdb.h |  4 +++
 libraries/liblmdb/mdb.c  | 74 ++++++++++++++++++++--------------------
 libraries/liblmdb/midl.c |  4 +--
 libraries/liblmdb/midl.h |  4 +--
 4 files changed, 45 insertions(+), 41 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 039210dcde..642b62d742 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -187,6 +187,10 @@ typedef	mode_t	mdb_mode_t;
 # define MDB_FMT_Z	"z"			/**< printf/scanf format modifier for size_t */
 #endif
 
+#if !defined(MDB_RPAGE_CACHE) || (defined(MDB_VL32) && !(MDB_RPAGE_CACHE))
+#define MDB_RPAGE_CACHE	1
+#endif
+
 #ifndef MDB_VL32
 /** Unsigned type used for mapsize, entry counts and page/transaction IDs.
  *
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index a64f60a140..33be6038f5 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1246,7 +1246,7 @@ struct MDB_txn {
 	/** Nested txn under this txn, set together with flag #MDB_TXN_HAS_CHILD */
 	MDB_txn		*mt_child;
 	pgno_t		mt_next_pgno;	/**< next unallocated page */
-#ifdef MDB_VL32
+#if MDB_RPAGE_CACHE
 	pgno_t		mt_last_pgno;	/**< last written page */
 #endif
 	/** The ID of this transaction. IDs are integers incrementing from 1.
@@ -1296,7 +1296,7 @@ struct MDB_txn {
 	MDB_cursor	**mt_cursors;
 	/** Array of flags for each DB */
 	unsigned char	*mt_dbflags;
-#ifdef MDB_VL32
+#if MDB_RPAGE_CACHE
 	/** List of read-only pages (actually chunks) */
 	MDB_ID3L	mt_rpages;
 	/** We map chunks of 16 pages. Even though Windows uses 4KB pages, all
@@ -1398,7 +1398,7 @@ struct MDB_cursor {
 	unsigned int	mc_flags;	/**< @ref mdb_cursor */
 	MDB_page	*mc_pg[CURSOR_STACK];	/**< stack of pushed pages */
 	indx_t		mc_ki[CURSOR_STACK];	/**< stack of page indices */
-#ifdef MDB_VL32
+#if MDB_RPAGE_CACHE
 	MDB_page	*mc_ovpg;		/**< a referenced overflow page */
 #	define MC_OVPG(mc)			((mc)->mc_ovpg)
 #	define MC_SET_OVPG(mc, pg)	((mc)->mc_ovpg = (pg))
@@ -1450,7 +1450,7 @@ struct MDB_env {
 	HANDLE		me_fd;		/**< The main data file */
 	HANDLE		me_lfd;		/**< The lock file */
 	HANDLE		me_mfd;		/**< For writing and syncing the meta pages */
-#if defined(MDB_VL32) && defined(_WIN32)
+#if MDB_RPAGE_CACHE && defined(_WIN32)
 	HANDLE		me_fmh;		/**< File Mapping handle */
 #endif
 	/** Failed to update the meta page. Probably an I/O error. */
@@ -1515,7 +1515,7 @@ struct MDB_env {
 	char		me_mutexname[sizeof(MUTEXNAME_PREFIX) + 11];
 # endif
 #endif
-#ifdef MDB_VL32
+#if MDB_RPAGE_CACHE
 	MDB_ID3L	me_rpages;	/**< like #mt_rpages, but global to env */
 	pthread_mutex_t	me_rpmutex;	/**< control access to #me_rpages */
 #define MDB_ERPAGE_SIZE	16384
@@ -2050,7 +2050,7 @@ mdb_dlist_free(MDB_txn *txn)
 	dl[0].mid = 0;
 }
 
-#ifdef MDB_VL32
+#if MDB_RPAGE_CACHE
 static void
 mdb_page_unref(MDB_txn *txn, MDB_page *mp)
 {
@@ -2095,7 +2095,7 @@ mdb_cursor_unref(MDB_cursor *mc)
 #else
 #define MDB_PAGE_UNREF(txn, mp)
 #define MDB_CURSOR_UNREF(mc, force) ((void)0)
-#endif /* MDB_VL32 */
+#endif /* MDB_RPAGE_CACHE */
 
 /** Loosen or free a single page.
  * Saves single pages to a list for future reuse
@@ -2554,7 +2554,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
 			rc = MDB_MAP_FULL;
 			goto fail;
 	}
-#if defined(_WIN32) && !defined(MDB_VL32)
+#if defined(_WIN32) && !MDB_RPAGE_CACHE
 	if (!(env->me_flags & MDB_RDONLY)) {
 		void *p;
 		p = (MDB_page *)(env->me_map + env->me_psize * pgno);
@@ -3067,7 +3067,7 @@ mdb_txn_renew0(MDB_txn *txn)
 
 	/* Moved to here to avoid a data race in read TXNs */
 	txn->mt_next_pgno = meta->mm_last_pg+1;
-#ifdef MDB_VL32
+#if MDB_RPAGE_CACHE
 	txn->mt_last_pgno = txn->mt_next_pgno - 1;
 #endif
 
@@ -3148,7 +3148,7 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 		DPRINTF(("calloc: %s", strerror(errno)));
 		return ENOMEM;
 	}
-#ifdef MDB_VL32
+#if MDB_RPAGE_CACHE
 	if (!parent) {
 		txn->mt_rpages = malloc(MDB_TRPAGE_SIZE * sizeof(MDB_ID3));
 		if (!txn->mt_rpages) {
@@ -3186,7 +3186,7 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 		parent->mt_child = txn;
 		txn->mt_parent = parent;
 		txn->mt_numdbs = parent->mt_numdbs;
-#ifdef MDB_VL32
+#if MDB_RPAGE_CACHE
 		txn->mt_rpages = parent->mt_rpages;
 #endif
 		memcpy(txn->mt_dbs, parent->mt_dbs, txn->mt_numdbs * sizeof(MDB_db));
@@ -3215,7 +3215,7 @@ renew:
 	}
 	if (rc) {
 		if (txn != env->me_txn0) {
-#ifdef MDB_VL32
+#if MDB_RPAGE_CACHE
 			free(txn->mt_rpages);
 #endif
 			free(txn);
@@ -3344,7 +3344,7 @@ mdb_txn_end(MDB_txn *txn, unsigned mode)
 
 		mdb_midl_free(pghead);
 	}
-#ifdef MDB_VL32
+#if MDB_RPAGE_CACHE
 	if (!txn->mt_parent) {
 		MDB_ID3L el = env->me_rpages, tl = txn->mt_rpages;
 		unsigned i, x, n = tl[0].mid;
@@ -3721,7 +3721,7 @@ retry_seek:
 		n++;
 #endif	/* _WIN32 */
 	}
-#ifdef MDB_VL32
+#if MDB_RPAGE_CACHE
 	if (pgno > txn->mt_last_pgno)
 		txn->mt_last_pgno = pgno;
 #endif
@@ -4327,11 +4327,11 @@ mdb_env_map(MDB_env *env, void *addr)
 	if (rc)
 		return mdb_nt2win32(rc);
 	map = addr;
-#ifdef MDB_VL32
+#if MDB_RPAGE_CACHE
 	msize = NUM_METAS * env->me_psize;
 #endif
 	rc = NtMapViewOfSection(mh, GetCurrentProcess(), &map, 0, 0, NULL, &msize, ViewUnmap, alloctype, pageprot);
-#ifdef MDB_VL32
+#if MDB_RPAGE_CACHE
 	env->me_fmh = mh;
 #else
 	NtClose(mh);
@@ -4340,7 +4340,7 @@ mdb_env_map(MDB_env *env, void *addr)
 		return mdb_nt2win32(rc);
 	env->me_map = map;
 #else
-#ifdef MDB_VL32
+#if MDB_RPAGE_CACHE
 	(void) flags;
 	env->me_map = mmap(addr, NUM_METAS * env->me_psize, PROT_READ, MAP_SHARED,
 		env->me_fd, 0);
@@ -4398,7 +4398,7 @@ mdb_env_set_mapsize(MDB_env *env, mdb_size_t size)
 	 */
 	if (env->me_map) {
 		MDB_meta *meta;
-#ifndef MDB_VL32
+#if !MDB_RPAGE_CACHE
 		void *old;
 		int rc;
 #endif
@@ -4413,8 +4413,8 @@ mdb_env_set_mapsize(MDB_env *env, mdb_size_t size)
 			if (size < minsize)
 				size = minsize;
 		}
-#ifndef MDB_VL32
-		/* For MDB_VL32 this bit is a noop since we dynamically remap
+#if !MDB_RPAGE_CACHE
+		/* For MDB_RPAGE_CACHE this bit is a noop since we dynamically remap
 		 * chunks of the DB anyway.
 		 */
 		munmap(env->me_map, env->me_mapsize);
@@ -4423,7 +4423,7 @@ mdb_env_set_mapsize(MDB_env *env, mdb_size_t size)
 		rc = mdb_env_map(env, old);
 		if (rc)
 			return rc;
-#endif /* !MDB_VL32 */
+#endif /* !MDB_RPAGE_CACHE */
 	}
 	env->me_mapsize = size;
 	if (env->me_psize)
@@ -5347,9 +5347,9 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 	if (env->me_fd!=INVALID_HANDLE_VALUE || (flags & ~(CHANGEABLE|CHANGELESS)))
 		return EINVAL;
 
-#ifdef MDB_VL32
+#if MDB_RPAGE_CACHE
 	if (flags & MDB_WRITEMAP) {
-		/* silently ignore WRITEMAP in 32 bit mode */
+		/* silently ignore WRITEMAP with RPAGE_CACHE */
 		flags ^= MDB_WRITEMAP;
 	}
 	if (flags & MDB_FIXEDMAP) {
@@ -5363,7 +5363,7 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 	if (rc)
 		return rc;
 
-#ifdef MDB_VL32
+#if MDB_RPAGE_CACHE
 #ifdef _WIN32
 	env->me_rpmutex = CreateMutex(NULL, FALSE, NULL);
 	if (!env->me_rpmutex) {
@@ -5391,7 +5391,7 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 	if (rc)
 		goto leave;
 
-#ifdef MDB_VL32
+#if MDB_RPAGE_CACHE
 	{
 		env->me_rpages = malloc(MDB_ERPAGE_SIZE * sizeof(MDB_ID3));
 		if (!env->me_rpages) {
@@ -5459,7 +5459,7 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 				txn->mt_dbiseqs = (unsigned int *)(txn->mt_cursors + env->me_maxdbs);
 				txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs);
 				txn->mt_env = env;
-#ifdef MDB_VL32
+#if MDB_RPAGE_CACHE
 				txn->mt_rpages = malloc(MDB_TRPAGE_SIZE * sizeof(MDB_ID3));
 				if (!txn->mt_rpages) {
 					free(txn);
@@ -5507,7 +5507,7 @@ mdb_env_close0(MDB_env *env, int excl)
 	free(env->me_dbflags);
 	free(env->me_path);
 	free(env->me_dirty_list);
-#ifdef MDB_VL32
+#if MDB_RPAGE_CACHE
 	if (env->me_txn0 && env->me_txn0->mt_rpages)
 		free(env->me_txn0->mt_rpages);
 	if (env->me_rpages) {
@@ -5535,7 +5535,7 @@ mdb_env_close0(MDB_env *env, int excl)
 	}
 
 	if (env->me_map) {
-#ifdef MDB_VL32
+#if MDB_RPAGE_CACHE
 		munmap(env->me_map, NUM_METAS*env->me_psize);
 #else
 		munmap(env->me_map, env->me_mapsize);
@@ -5604,7 +5604,7 @@ mdb_env_close0(MDB_env *env, int excl)
 #endif
 		(void) close(env->me_lfd);
 	}
-#ifdef MDB_VL32
+#if MDB_RPAGE_CACHE
 #ifdef _WIN32
 	if (env->me_fmh) CloseHandle(env->me_fmh);
 	if (env->me_rpmutex) CloseHandle(env->me_rpmutex);
@@ -5880,7 +5880,7 @@ mdb_cursor_push(MDB_cursor *mc, MDB_page *mp)
 	return MDB_SUCCESS;
 }
 
-#ifdef MDB_VL32
+#if MDB_RPAGE_CACHE
 /** Map a read-only page.
  * There are two levels of tracking in use, a per-txn list and a per-env list.
  * ref'ing and unref'ing the per-txn list is faster since it requires no
@@ -6222,7 +6222,7 @@ mdb_page_get(MDB_cursor *mc, pgno_t pgno, MDB_page **ret, int *lvl)
 
 mapped:
 	{
-#ifdef MDB_VL32
+#if MDB_RPAGE_CACHE
 		int rc = mdb_rpage_get(txn, pgno, &p);
 		if (rc) {
 			txn->mt_flags |= MDB_TXN_ERROR;
@@ -6413,7 +6413,7 @@ mdb_page_search(MDB_cursor *mc, MDB_val *key, int flags)
 
 	mdb_cassert(mc, root > 1);
 	if (!mc->mc_pg[0] || mc->mc_pg[0]->mp_pgno != root) {
-#ifdef MDB_VL32
+#if MDB_RPAGE_CACHE
 		if (mc->mc_pg[0])
 			MDB_PAGE_UNREF(mc->mc_txn, mc->mc_pg[0]);
 #endif
@@ -6421,7 +6421,7 @@ mdb_page_search(MDB_cursor *mc, MDB_val *key, int flags)
 			return rc;
 	}
 
-#ifdef MDB_VL32
+#if MDB_RPAGE_CACHE
 	{
 		int i;
 		for (i=1; i<mc->mc_snum; i++)
@@ -6577,7 +6577,7 @@ mdb_get(MDB_txn *txn, MDB_dbi dbi,
 
 	mdb_cursor_init(&mc, txn, dbi, &mx);
 	rc = mdb_cursor_set(&mc, key, data, MDB_SET, &exact);
-	/* unref all the pages when MDB_VL32 - caller must copy the data
+	/* unref all the pages when MDB_RPAGE_CACHE - caller must copy the data
 	 * before doing anything else
 	 */
 	MDB_CURSOR_UNREF(&mc, 1);
@@ -6598,7 +6598,7 @@ mdb_cursor_sibling(MDB_cursor *mc, int move_right)
 	int		 rc;
 	MDB_node	*indx;
 	MDB_page	*mp;
-#ifdef MDB_VL32
+#if MDB_RPAGE_CACHE
 	MDB_page	*op;
 #endif
 
@@ -6606,7 +6606,7 @@ mdb_cursor_sibling(MDB_cursor *mc, int move_right)
 		return MDB_NOTFOUND;		/* root has no siblings */
 	}
 
-#ifdef MDB_VL32
+#if MDB_RPAGE_CACHE
 	op = mc->mc_pg[mc->mc_top];
 #endif
 	mdb_cursor_pop(mc);
@@ -10723,7 +10723,7 @@ mdb_drop0(MDB_cursor *mc, int subs)
 			mdb_cursor_pop(mc);
 
 		mdb_cursor_copy(mc, &mx);
-#ifdef MDB_VL32
+#if MDB_RPAGE_CACHE
 		/* bump refcount for mx's pages */
 		for (i=0; i<mc->mc_snum; i++)
 			mdb_page_get(&mx, mc->mc_pg[i]->mp_pgno, &mx.mc_pg[i], NULL);
diff --git a/libraries/liblmdb/midl.c b/libraries/liblmdb/midl.c
index 499df17ccb..05b91f7ed4 100644
--- a/libraries/liblmdb/midl.c
+++ b/libraries/liblmdb/midl.c
@@ -355,7 +355,7 @@ int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id )
 	return 0;
 }
 
-#ifdef MDB_VL32
+#if MDB_RPAGE_CACHE
 unsigned mdb_mid3l_search( MDB_ID3L ids, MDB_ID id )
 {
 	/*
@@ -415,7 +415,7 @@ int mdb_mid3l_insert( MDB_ID3L ids, MDB_ID3 *id )
 
 	return 0;
 }
-#endif /* MDB_VL32 */
+#endif /* MDB_RPAGE_CACHE */
 
 /** @} */
 /** @} */
diff --git a/libraries/liblmdb/midl.h b/libraries/liblmdb/midl.h
index 48591dd772..ac6bdd0ecb 100644
--- a/libraries/liblmdb/midl.h
+++ b/libraries/liblmdb/midl.h
@@ -178,7 +178,7 @@ int mdb_mid2l_insert( MDB_ID2L ids, MDB_ID2 *id );
 	 */
 int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id );
 
-#ifdef MDB_VL32
+#if MDB_RPAGE_CACHE
 typedef struct MDB_ID3 {
 	MDB_ID mid;		/**< The ID */
 	void *mptr;		/**< The pointer */
@@ -191,7 +191,7 @@ typedef MDB_ID3 *MDB_ID3L;
 unsigned mdb_mid3l_search( MDB_ID3L ids, MDB_ID id );
 int mdb_mid3l_insert( MDB_ID3L ids, MDB_ID3 *id );
 
-#endif /* MDB_VL32 */
+#endif /* MDB_RPAGE_CACHE */
 /** @} */
 /** @} */
 #ifdef __cplusplus

From 53799e51da192e970f7630a07fb11a4047a1c9d2 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 1 Jun 2017 17:48:04 +0100
Subject: [PATCH 307/504] RPAGE_CACHE is now dynamically selectable

Behavior is controlled by MDB_REMAP_CHUNKS envflag
Remapping is always enabled in MDB_VL32
---
 libraries/liblmdb/lmdb.h |   3 +
 libraries/liblmdb/mdb.c  | 134 +++++++++++++++++++++++----------------
 2 files changed, 82 insertions(+), 55 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 642b62d742..7eb0a1ea16 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -188,6 +188,7 @@ typedef	mode_t	mdb_mode_t;
 #endif
 
 #if !defined(MDB_RPAGE_CACHE) || (defined(MDB_VL32) && !(MDB_RPAGE_CACHE))
+/** Support #MDB_REMAP_CHUNKS. Implied by MDB_VL32. Define as 0 to disable. */
 #define MDB_RPAGE_CACHE	1
 #endif
 
@@ -336,6 +337,8 @@ typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *rel
 #define MDB_NORDAHEAD	0x800000
 	/** don't initialize malloc'd memory before writing to datafile */
 #define MDB_NOMEMINIT	0x1000000
+	/** don't use a single mmap, remap individual chunks (needs MDB_RPAGE_CACHE) */
+#define MDB_REMAP_CHUNKS	0x2000000
 /** @} */
 
 /**	@defgroup	mdb_dbi_open	Database Flags
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 33be6038f5..ffc32a9107 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2051,6 +2051,12 @@ mdb_dlist_free(MDB_txn *txn)
 }
 
 #if MDB_RPAGE_CACHE
+#if defined(MDB_VL32) || ((MDB_RPAGE_CACHE) & 2) /* Always remap */
+#define MDB_REMAPPING(flags)	1
+#else
+#define MDB_REMAPPING(flags)	((flags) & MDB_REMAP_CHUNKS)
+#endif
+
 static void
 mdb_page_unref(MDB_txn *txn, MDB_page *mp)
 {
@@ -2067,7 +2073,8 @@ mdb_page_unref(MDB_txn *txn, MDB_page *mp)
 	if (tl[x].mref)
 		tl[x].mref--;
 }
-#define MDB_PAGE_UNREF(txn, mp)	mdb_page_unref(txn, mp)
+#define MDB_PAGE_UNREF(txn, mp) \
+	(MDB_REMAPPING(txn->mt_env->me_flags) ? mdb_page_unref(txn, mp) : (void)0)
 
 static void
 mdb_cursor_unref(MDB_cursor *mc)
@@ -2088,11 +2095,13 @@ mdb_cursor_unref(MDB_cursor *mc)
 	mc->mc_flags &= ~C_INITIALIZED;
 }
 #define MDB_CURSOR_UNREF(mc, force) \
-	(((force) || ((mc)->mc_flags & C_INITIALIZED)) \
+	((MDB_REMAPPING((mc)->mc_txn->mt_env->me_flags) && \
+	 ((force) || ((mc)->mc_flags & C_INITIALIZED))) \
 	 ? mdb_cursor_unref(mc) \
 	 : (void)0)
 
 #else
+#define MDB_REMAPPING(flags)	0
 #define MDB_PAGE_UNREF(txn, mp)
 #define MDB_CURSOR_UNREF(mc, force) ((void)0)
 #endif /* MDB_RPAGE_CACHE */
@@ -2554,8 +2563,8 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
 			rc = MDB_MAP_FULL;
 			goto fail;
 	}
-#if defined(_WIN32) && !MDB_RPAGE_CACHE
-	if (!(env->me_flags & MDB_RDONLY)) {
+#if defined(_WIN32)
+	if (!MDB_REMAPPING(env->me_flags) && !(env->me_flags & MDB_RDONLY)) {
 		void *p;
 		p = (MDB_page *)(env->me_map + env->me_psize * pgno);
 		p = VirtualAlloc(p, env->me_psize * num, MEM_COMMIT,
@@ -3068,7 +3077,8 @@ mdb_txn_renew0(MDB_txn *txn)
 	/* Moved to here to avoid a data race in read TXNs */
 	txn->mt_next_pgno = meta->mm_last_pg+1;
 #if MDB_RPAGE_CACHE
-	txn->mt_last_pgno = txn->mt_next_pgno - 1;
+	if (MDB_REMAPPING(env->me_flags))
+		txn->mt_last_pgno = txn->mt_next_pgno - 1;
 #endif
 
 	txn->mt_flags = flags;
@@ -3149,7 +3159,7 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 		return ENOMEM;
 	}
 #if MDB_RPAGE_CACHE
-	if (!parent) {
+	if (MDB_REMAPPING(env->me_flags) && !parent) {
 		txn->mt_rpages = malloc(MDB_TRPAGE_SIZE * sizeof(MDB_ID3));
 		if (!txn->mt_rpages) {
 			free(txn);
@@ -3187,7 +3197,8 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 		txn->mt_parent = parent;
 		txn->mt_numdbs = parent->mt_numdbs;
 #if MDB_RPAGE_CACHE
-		txn->mt_rpages = parent->mt_rpages;
+		if (MDB_REMAPPING(env->me_flags))
+			txn->mt_rpages = parent->mt_rpages;
 #endif
 		memcpy(txn->mt_dbs, parent->mt_dbs, txn->mt_numdbs * sizeof(MDB_db));
 		/* Copy parent's mt_dbflags, but clear DB_NEW */
@@ -3216,7 +3227,8 @@ renew:
 	if (rc) {
 		if (txn != env->me_txn0) {
 #if MDB_RPAGE_CACHE
-			free(txn->mt_rpages);
+			if (MDB_REMAPPING(env->me_flags))
+				free(txn->mt_rpages);
 #endif
 			free(txn);
 		}
@@ -3345,7 +3357,7 @@ mdb_txn_end(MDB_txn *txn, unsigned mode)
 		mdb_midl_free(pghead);
 	}
 #if MDB_RPAGE_CACHE
-	if (!txn->mt_parent) {
+	if (MDB_REMAPPING(env->me_flags) && !txn->mt_parent) {
 		MDB_ID3L el = env->me_rpages, tl = txn->mt_rpages;
 		unsigned i, x, n = tl[0].mid;
 		pthread_mutex_lock(&env->me_rpmutex);
@@ -3722,7 +3734,7 @@ retry_seek:
 #endif	/* _WIN32 */
 	}
 #if MDB_RPAGE_CACHE
-	if (pgno > txn->mt_last_pgno)
+	if (MDB_REMAPPING(env->me_flags) && pgno > txn->mt_last_pgno)
 		txn->mt_last_pgno = pgno;
 #endif
 
@@ -4327,28 +4339,29 @@ mdb_env_map(MDB_env *env, void *addr)
 	if (rc)
 		return mdb_nt2win32(rc);
 	map = addr;
-#if MDB_RPAGE_CACHE
-	msize = NUM_METAS * env->me_psize;
-#endif
+	if (MDB_REMAPPING(env->me_flags))
+		msize = NUM_METAS * env->me_psize;
 	rc = NtMapViewOfSection(mh, GetCurrentProcess(), &map, 0, 0, NULL, &msize, ViewUnmap, alloctype, pageprot);
 #if MDB_RPAGE_CACHE
-	env->me_fmh = mh;
-#else
-	NtClose(mh);
+	if (MDB_REMAPPING(env->me_flags))
+		env->me_fmh = mh;
+	else
 #endif
+		NtClose(mh);
 	if (rc)
 		return mdb_nt2win32(rc);
 	env->me_map = map;
-#else
-#if MDB_RPAGE_CACHE
-	(void) flags;
-	env->me_map = mmap(addr, NUM_METAS * env->me_psize, PROT_READ, MAP_SHARED,
-		env->me_fd, 0);
-	if (env->me_map == MAP_FAILED) {
-		env->me_map = NULL;
-		return ErrCode();
-	}
-#else
+#else	/* !_WIN32 */
+	if (MDB_REMAPPING(env->me_flags)) {
+		(void) flags;
+		env->me_map = mmap(addr, NUM_METAS * env->me_psize, PROT_READ, MAP_SHARED,
+			env->me_fd, 0);
+		if (env->me_map == MAP_FAILED) {
+			env->me_map = NULL;
+			return ErrCode();
+		}
+	} else
+	{
 	int prot = PROT_READ;
 	if (flags & MDB_WRITEMAP) {
 		prot |= PROT_WRITE;
@@ -4372,6 +4385,7 @@ mdb_env_map(MDB_env *env, void *addr)
 #endif /* POSIX_MADV_RANDOM */
 #endif /* MADV_RANDOM */
 	}
+	}
 #endif /* _WIN32 */
 
 	/* Can happen because the address argument to mmap() is just a
@@ -4381,7 +4395,6 @@ mdb_env_map(MDB_env *env, void *addr)
 	 */
 	if (addr && env->me_map != addr)
 		return EBUSY;	/* TODO: Make a new MDB_* error code? */
-#endif
 
 	p = (MDB_page *)env->me_map;
 	env->me_metas[0] = METADATA(p);
@@ -4398,10 +4411,9 @@ mdb_env_set_mapsize(MDB_env *env, mdb_size_t size)
 	 */
 	if (env->me_map) {
 		MDB_meta *meta;
-#if !MDB_RPAGE_CACHE
 		void *old;
 		int rc;
-#endif
+
 		if (env->me_txn)
 			return EINVAL;
 		meta = mdb_env_pick_meta(env);
@@ -4413,8 +4425,9 @@ mdb_env_set_mapsize(MDB_env *env, mdb_size_t size)
 			if (size < minsize)
 				size = minsize;
 		}
-#if !MDB_RPAGE_CACHE
-		/* For MDB_RPAGE_CACHE this bit is a noop since we dynamically remap
+		if (!(MDB_REMAPPING(env->me_flags)))
+		{
+		/* For MDB_REMAP_CHUNKS this bit is a noop since we dynamically remap
 		 * chunks of the DB anyway.
 		 */
 		munmap(env->me_map, env->me_mapsize);
@@ -4423,7 +4436,7 @@ mdb_env_set_mapsize(MDB_env *env, mdb_size_t size)
 		rc = mdb_env_map(env, old);
 		if (rc)
 			return rc;
-#endif /* !MDB_RPAGE_CACHE */
+		}
 	}
 	env->me_mapsize = size;
 	if (env->me_psize)
@@ -5332,7 +5345,7 @@ fail:
 	 */
 #define	CHANGEABLE	(MDB_NOSYNC|MDB_NOMETASYNC|MDB_MAPASYNC|MDB_NOMEMINIT)
 #define	CHANGELESS	(MDB_FIXEDMAP|MDB_NOSUBDIR|MDB_RDONLY| \
-	MDB_WRITEMAP|MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD)
+	MDB_WRITEMAP|MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD|MDB_REMAP_CHUNKS)
 
 #if VALID_FLAGS & PERSISTENT_FLAGS & (CHANGEABLE|CHANGELESS)
 # error "Persistent DB flags & env flags overlap, but both go in mm_flags"
@@ -5347,23 +5360,23 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 	if (env->me_fd!=INVALID_HANDLE_VALUE || (flags & ~(CHANGEABLE|CHANGELESS)))
 		return EINVAL;
 
-#if MDB_RPAGE_CACHE
-	if (flags & MDB_WRITEMAP) {
-		/* silently ignore WRITEMAP with RPAGE_CACHE */
-		flags ^= MDB_WRITEMAP;
-	}
-	if (flags & MDB_FIXEDMAP) {
-		/* cannot support FIXEDMAP */
-		return EINVAL;
-	}
-#endif
 	flags |= env->me_flags;
+	if (MDB_REMAPPING(0))			/* if we always remap chunks */
+		flags |= MDB_REMAP_CHUNKS;
+	if (MDB_REMAPPING(flags)) {
+		/* silently ignore WRITEMAP with REMAP_CHUNKS */
+		flags &= ~MDB_WRITEMAP;
+		/* cannot support FIXEDMAP */
+		if (flags & MDB_FIXEDMAP)
+			return EINVAL;
+	}
 
 	rc = mdb_fname_init(path, flags, &fname);
 	if (rc)
 		return rc;
 
 #if MDB_RPAGE_CACHE
+	if (MDB_REMAPPING(flags)) {
 #ifdef _WIN32
 	env->me_rpmutex = CreateMutex(NULL, FALSE, NULL);
 	if (!env->me_rpmutex) {
@@ -5375,6 +5388,7 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 	if (rc)
 		goto leave;
 #endif
+	}
 #endif
 	flags |= MDB_ENV_ACTIVE;	/* tell mdb_env_close0() to clean up */
 
@@ -5392,6 +5406,7 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 		goto leave;
 
 #if MDB_RPAGE_CACHE
+	if (MDB_REMAPPING(flags))
 	{
 		env->me_rpages = malloc(MDB_ERPAGE_SIZE * sizeof(MDB_ID3));
 		if (!env->me_rpages) {
@@ -5460,6 +5475,7 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 				txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs);
 				txn->mt_env = env;
 #if MDB_RPAGE_CACHE
+				if (MDB_REMAPPING(env->me_flags)) {
 				txn->mt_rpages = malloc(MDB_TRPAGE_SIZE * sizeof(MDB_ID3));
 				if (!txn->mt_rpages) {
 					free(txn);
@@ -5468,6 +5484,7 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 				}
 				txn->mt_rpages[0].mid = 0;
 				txn->mt_rpcheck = MDB_TRPAGE_SIZE/2;
+				}
 #endif
 				txn->mt_dbxs = env->me_dbxs;
 				txn->mt_flags = MDB_TXN_FINISHED;
@@ -5508,6 +5525,7 @@ mdb_env_close0(MDB_env *env, int excl)
 	free(env->me_path);
 	free(env->me_dirty_list);
 #if MDB_RPAGE_CACHE
+	if (MDB_REMAPPING(env->me_flags)) {
 	if (env->me_txn0 && env->me_txn0->mt_rpages)
 		free(env->me_txn0->mt_rpages);
 	if (env->me_rpages) {
@@ -5517,6 +5535,7 @@ mdb_env_close0(MDB_env *env, int excl)
 			munmap(el[x].mptr, el[x].mcnt * env->me_psize);
 		free(el);
 	}
+	}
 #endif
 	free(env->me_txn0);
 	mdb_midl_free(env->me_free_pgs);
@@ -5535,11 +5554,10 @@ mdb_env_close0(MDB_env *env, int excl)
 	}
 
 	if (env->me_map) {
-#if MDB_RPAGE_CACHE
-		munmap(env->me_map, NUM_METAS*env->me_psize);
-#else
-		munmap(env->me_map, env->me_mapsize);
-#endif
+		if (MDB_REMAPPING(env->me_flags))
+			munmap(env->me_map, NUM_METAS*env->me_psize);
+		else
+			munmap(env->me_map, env->me_mapsize);
 	}
 	if (env->me_mfd != INVALID_HANDLE_VALUE)
 		(void) close(env->me_mfd);
@@ -5605,12 +5623,15 @@ mdb_env_close0(MDB_env *env, int excl)
 		(void) close(env->me_lfd);
 	}
 #if MDB_RPAGE_CACHE
+	if (MDB_REMAPPING(env->me_flags))
+	{
 #ifdef _WIN32
 	if (env->me_fmh) CloseHandle(env->me_fmh);
 	if (env->me_rpmutex) CloseHandle(env->me_rpmutex);
 #else
 	pthread_mutex_destroy(&env->me_rpmutex);
 #endif
+	}
 #endif
 
 	env->me_flags &= ~(MDB_ENV_ACTIVE|MDB_ENV_TXKEY);
@@ -6221,17 +6242,18 @@ mdb_page_get(MDB_cursor *mc, pgno_t pgno, MDB_page **ret, int *lvl)
 	level = 0;
 
 mapped:
-	{
 #if MDB_RPAGE_CACHE
+	if (MDB_REMAPPING(txn->mt_env->me_flags)) {
 		int rc = mdb_rpage_get(txn, pgno, &p);
 		if (rc) {
 			txn->mt_flags |= MDB_TXN_ERROR;
 			return rc;
 		}
-#else
+	} else
+#endif
+	{
 		MDB_env *env = txn->mt_env;
 		p = (MDB_page *)(env->me_map + env->me_psize * pgno);
-#endif
 	}
 
 done:
@@ -6422,10 +6444,11 @@ mdb_page_search(MDB_cursor *mc, MDB_val *key, int flags)
 	}
 
 #if MDB_RPAGE_CACHE
+	if (MDB_REMAPPING(mc->mc_txn->mt_env->me_flags))
 	{
 		int i;
 		for (i=1; i<mc->mc_snum; i++)
-			MDB_PAGE_UNREF(mc->mc_txn, mc->mc_pg[i]);
+			mdb_page_unref(mc->mc_txn, mc->mc_pg[i]);
 	}
 #endif
 	mc->mc_snum = 1;
@@ -6577,7 +6600,7 @@ mdb_get(MDB_txn *txn, MDB_dbi dbi,
 
 	mdb_cursor_init(&mc, txn, dbi, &mx);
 	rc = mdb_cursor_set(&mc, key, data, MDB_SET, &exact);
-	/* unref all the pages when MDB_RPAGE_CACHE - caller must copy the data
+	/* unref all the pages when MDB_REMAP_CHUNKS - caller must copy the data
 	 * before doing anything else
 	 */
 	MDB_CURSOR_UNREF(&mc, 1);
@@ -10723,11 +10746,12 @@ mdb_drop0(MDB_cursor *mc, int subs)
 			mdb_cursor_pop(mc);
 
 		mdb_cursor_copy(mc, &mx);
-#if MDB_RPAGE_CACHE
+		if (MDB_REMAPPING(mc->mc_txn->mt_env->me_flags)) {
 		/* bump refcount for mx's pages */
 		for (i=0; i<mc->mc_snum; i++)
 			mdb_page_get(&mx, mc->mc_pg[i]->mp_pgno, &mx.mc_pg[i], NULL);
-#endif
+		}
+
 		while (mc->mc_snum > 0) {
 			MDB_page *mp = mc->mc_pg[mc->mc_top];
 			unsigned n = NUMKEYS(mp);

From ad7933ba0c2457c918d798fef53210ffb2f11771 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 29 Jun 2017 16:07:17 +0100
Subject: [PATCH 308/504] Optimization for mdb_rpage_get()

The caller already knows if it's using an overflow page, so
pass the number of expected pages in instead of having to
map the page first and check the count there.
---
 libraries/liblmdb/mdb.c | 89 +++++++++++++++++++++++------------------
 1 file changed, 49 insertions(+), 40 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index ffc32a9107..8db0b80046 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1567,7 +1567,14 @@ enum {
 #define MDB_END_SLOT MDB_NOTLS	/**< release any reader slot if #MDB_NOTLS */
 static void mdb_txn_end(MDB_txn *txn, unsigned mode);
 
+#if MDB_RPAGE_CACHE
+static int  mdb_page_get(MDB_cursor *mc, pgno_t pgno, int numpgs, MDB_page **mp, int *lvl);
+#define MDB_PAGE_GET(mc, pg, numpgs, mp, lvl)	mdb_page_get(mc, pg, numpgs, mp, lvl)
+#else
 static int  mdb_page_get(MDB_cursor *mc, pgno_t pgno, MDB_page **mp, int *lvl);
+#define MDB_PAGE_GET(mc, pg, numpgs, mp, lvl)	mdb_page_get(mc, pg, mp, lvl)
+#endif
+
 static int  mdb_page_search_root(MDB_cursor *mc,
 			    MDB_val *key, int modify);
 #define MDB_PS_MODIFY	1
@@ -2215,7 +2222,7 @@ mark_done:
 				pgno_t pgno = txn->mt_dbs[i].md_root;
 				if (pgno == P_INVALID)
 					continue;
-				if ((rc = mdb_page_get(m0, pgno, &dp, &level)) != MDB_SUCCESS)
+				if ((rc = MDB_PAGE_GET(m0, pgno, 1, &dp, &level)) != MDB_SUCCESS)
 					break;
 				if ((dp->mp_flags & Mask) == pflags && level <= 1)
 					dp->mp_flags ^= P_KEEP;
@@ -5952,7 +5959,7 @@ mdb_cursor_push(MDB_cursor *mc, MDB_page *mp)
  * @return 0 on success, non-zero on failure.
  */
 static int
-mdb_rpage_get(MDB_txn *txn, pgno_t pg0, MDB_page **ret)
+mdb_rpage_get(MDB_txn *txn, pgno_t pg0, int numpgs, MDB_page **ret)
 {
 	MDB_env *env = txn->mt_env;
 	MDB_page *p;
@@ -5992,9 +5999,8 @@ mdb_rpage_get(MDB_txn *txn, pgno_t pg0, MDB_page **ret)
 		if (x != tl[0].mid && tl[x+1].mid == pg0)
 			x++;
 		/* check for overflow size */
-		p = (MDB_page *)((char *)tl[x].mptr + rem * env->me_psize);
-		if (IS_OVERFLOW(p) && p->mp_pages + rem > tl[x].mcnt) {
-			id3.mcnt = p->mp_pages + rem;
+		if (numpgs + rem > tl[x].mcnt) {
+			id3.mcnt = numpgs + rem;
 			len = id3.mcnt * env->me_psize;
 			SET_OFF(off, pgno * env->me_psize);
 			MAP(rc, env, id3.mptr, len, off);
@@ -6088,7 +6094,7 @@ retry:
 		if ((env->me_flags & MDB_RDONLY) && pgno + MDB_RPAGE_CHUNK-1 > txn->mt_last_pgno)
 			id3.mcnt = txn->mt_last_pgno + 1 - pgno;
 		else
-			id3.mcnt = MDB_RPAGE_CHUNK;
+			id3.mcnt = numpgs + rem > MDB_RPAGE_CHUNK ? numpgs + rem : MDB_RPAGE_CHUNK;
 		len = id3.mcnt * env->me_psize;
 		id3.mid = pgno;
 
@@ -6097,12 +6103,8 @@ retry:
 		x = mdb_mid3l_search(el, pgno);
 		if (x <= el[0].mid && el[x].mid == pgno) {
 			id3.mptr = el[x].mptr;
-			id3.mcnt = el[x].mcnt;
 			/* check for overflow size */
-			p = (MDB_page *)((char *)id3.mptr + rem * env->me_psize);
-			if (IS_OVERFLOW(p) && p->mp_pages + rem > id3.mcnt) {
-				id3.mcnt = p->mp_pages + rem;
-				len = id3.mcnt * env->me_psize;
+			if (id3.mcnt > el[x].mcnt) {
 				SET_OFF(off, pgno * env->me_psize);
 				MAP(rc, env, id3.mptr, len, off);
 				if (rc)
@@ -6160,16 +6162,6 @@ fail:
 			pthread_mutex_unlock(&env->me_rpmutex);
 			return rc;
 		}
-		/* check for overflow size */
-		p = (MDB_page *)((char *)id3.mptr + rem * env->me_psize);
-		if (IS_OVERFLOW(p) && p->mp_pages + rem > id3.mcnt) {
-			id3.mcnt = p->mp_pages + rem;
-			munmap(id3.mptr, len);
-			len = id3.mcnt * env->me_psize;
-			MAP(rc, env, id3.mptr, len, off);
-			if (rc)
-				goto fail;
-		}
 		mdb_mid3l_insert(el, &id3);
 		pthread_mutex_unlock(&env->me_rpmutex);
 found:
@@ -6198,7 +6190,11 @@ ok:
  * @return 0 on success, non-zero on failure.
  */
 static int
-mdb_page_get(MDB_cursor *mc, pgno_t pgno, MDB_page **ret, int *lvl)
+mdb_page_get(MDB_cursor *mc, pgno_t pgno,
+#if MDB_RPAGE_CACHE
+	int numpgs,
+#endif
+	MDB_page **ret, int *lvl)
 {
 	MDB_txn *txn = mc->mc_txn;
 	MDB_page *p = NULL;
@@ -6244,7 +6240,7 @@ mdb_page_get(MDB_cursor *mc, pgno_t pgno, MDB_page **ret, int *lvl)
 mapped:
 #if MDB_RPAGE_CACHE
 	if (MDB_REMAPPING(txn->mt_env->me_flags)) {
-		int rc = mdb_rpage_get(txn, pgno, &p);
+		int rc = mdb_rpage_get(txn, pgno, numpgs, &p);
 		if (rc) {
 			txn->mt_flags |= MDB_TXN_ERROR;
 			return rc;
@@ -6316,7 +6312,7 @@ mdb_page_search_root(MDB_cursor *mc, MDB_val *key, int flags)
 		mdb_cassert(mc, i < NUMKEYS(mp));
 		node = NODEPTR(mp, i);
 
-		if ((rc = mdb_page_get(mc, NODEPGNO(node), &mp, NULL)) != 0)
+		if ((rc = MDB_PAGE_GET(mc, NODEPGNO(node), 1, &mp, NULL)) != 0)
 			return rc;
 
 		mc->mc_ki[mc->mc_top] = i;
@@ -6359,7 +6355,7 @@ mdb_page_search_lowest(MDB_cursor *mc)
 	MDB_node	*node = NODEPTR(mp, 0);
 	int rc;
 
-	if ((rc = mdb_page_get(mc, NODEPGNO(node), &mp, NULL)) != 0)
+	if ((rc = MDB_PAGE_GET(mc, NODEPGNO(node), 1, &mp, NULL)) != 0)
 		return rc;
 
 	mc->mc_ki[mc->mc_top] = 0;
@@ -6439,7 +6435,7 @@ mdb_page_search(MDB_cursor *mc, MDB_val *key, int flags)
 		if (mc->mc_pg[0])
 			MDB_PAGE_UNREF(mc->mc_txn, mc->mc_pg[0]);
 #endif
-		if ((rc = mdb_page_get(mc, root, &mc->mc_pg[0], NULL)) != 0)
+		if ((rc = MDB_PAGE_GET(mc, root, 1, &mc->mc_pg[0], NULL)) != 0)
 			return rc;
 	}
 
@@ -6571,10 +6567,15 @@ mdb_node_read(MDB_cursor *mc, MDB_node *leaf, MDB_val *data)
 	 */
 	data->mv_size = NODEDSZ(leaf);
 	memcpy(&pgno, NODEDATA(leaf), sizeof(pgno));
-	if ((rc = mdb_page_get(mc, pgno, &omp, NULL)) != 0) {
+	{
+#if MDB_RPAGE_CACHE
+	int dpages = OVPAGES(data->mv_size, mc->mc_txn->mt_env->me_psize);
+#endif
+	if ((rc = MDB_PAGE_GET(mc, pgno, dpages, &omp, NULL)) != 0) {
 		DPRINTF(("read overflow page %"Yu" failed", pgno));
 		return rc;
 	}
+	}
 	data->mv_data = METADATA(omp);
 	MC_SET_OVPG(mc, omp);
 
@@ -6659,7 +6660,7 @@ mdb_cursor_sibling(MDB_cursor *mc, int move_right)
 	MDB_PAGE_UNREF(mc->mc_txn, op);
 
 	indx = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
-	if ((rc = mdb_page_get(mc, NODEPGNO(indx), &mp, NULL)) != 0) {
+	if ((rc = MDB_PAGE_GET(mc, NODEPGNO(indx), 1, &mp, NULL)) != 0) {
 		/* mc will be inconsistent if caller does mc_snum++ as above */
 		mc->mc_flags &= ~(C_INITIALIZED|C_EOF);
 		return rc;
@@ -7678,7 +7679,7 @@ current:
 			int level, ovpages, dpages = OVPAGES(data->mv_size, env->me_psize);
 
 			memcpy(&pg, olddata.mv_data, sizeof(pg));
-			if ((rc2 = mdb_page_get(mc, pg, &omp, &level)) != 0)
+			if ((rc2 = MDB_PAGE_GET(mc, pg, dpages, &omp, &level)) != 0)
 				return rc2;
 			ovpages = omp->mp_pages;
 
@@ -7978,7 +7979,10 @@ mdb_cursor_del(MDB_cursor *mc, unsigned int flags)
 		pgno_t pg;
 
 		memcpy(&pg, NODEDATA(leaf), sizeof(pg));
-		if ((rc = mdb_page_get(mc, pg, &omp, NULL)) ||
+		/* note we don't care about page count here since
+		 * we're just adding pgno to the freelist anyway
+		 */
+		if ((rc = MDB_PAGE_GET(mc, pg, 1, &omp, NULL)) ||
 			(rc = mdb_ovpage_free(mc, omp)))
 			goto fail;
 	}
@@ -9141,7 +9145,7 @@ mdb_rebalance(MDB_cursor *mc)
 			if (rc)
 				return rc;
 			mc->mc_db->md_root = NODEPGNO(NODEPTR(mp, 0));
-			rc = mdb_page_get(mc, mc->mc_db->md_root, &mc->mc_pg[0], NULL);
+			rc = MDB_PAGE_GET(mc, mc->mc_db->md_root, 1, &mc->mc_pg[0], NULL);
 			if (rc)
 				return rc;
 			mc->mc_db->md_depth--;
@@ -9202,7 +9206,7 @@ mdb_rebalance(MDB_cursor *mc)
 		DPUTS("reading right neighbor");
 		mn.mc_ki[ptop]++;
 		node = NODEPTR(mc->mc_pg[ptop], mn.mc_ki[ptop]);
-		rc = mdb_page_get(mc, NODEPGNO(node), &mn.mc_pg[mn.mc_top], NULL);
+		rc = MDB_PAGE_GET(mc, NODEPGNO(node), 1, &mn.mc_pg[mn.mc_top], NULL);
 		if (rc)
 			return rc;
 		mn.mc_ki[mn.mc_top] = 0;
@@ -9214,7 +9218,7 @@ mdb_rebalance(MDB_cursor *mc)
 		DPUTS("reading left neighbor");
 		mn.mc_ki[ptop]--;
 		node = NODEPTR(mc->mc_pg[ptop], mn.mc_ki[ptop]);
-		rc = mdb_page_get(mc, NODEPGNO(node), &mn.mc_pg[mn.mc_top], NULL);
+		rc = MDB_PAGE_GET(mc, NODEPGNO(node), 1, &mn.mc_pg[mn.mc_top], NULL);
 		if (rc)
 			return rc;
 		mn.mc_ki[mn.mc_top] = NUMKEYS(mn.mc_pg[mn.mc_top]) - 1;
@@ -10008,7 +10012,7 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags)
 	mc.mc_txn = my->mc_txn;
 	mc.mc_flags = my->mc_txn->mt_flags & (C_ORIG_RDONLY|C_WRITEMAP);
 
-	rc = mdb_page_get(&mc, *pg, &mc.mc_pg[0], NULL);
+	rc = MDB_PAGE_GET(&mc, *pg, 1, &mc.mc_pg[0], NULL);
 	if (rc)
 		return rc;
 	rc = mdb_page_search_root(&mc, NULL, MDB_PS_FIRST);
@@ -10042,6 +10046,8 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags)
 					if (ni->mn_flags & F_BIGDATA) {
 						MDB_page *omp;
 						pgno_t pg;
+						size_t dsize;
+						int dpages;
 
 						/* Need writable leaf */
 						if (mp != leaf) {
@@ -10050,10 +10056,12 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags)
 							mp = leaf;
 							ni = NODEPTR(mp, i);
 						}
+						dsize = NODEDSZ(ni);
+						dpages = OVPAGES(dsize, my->mc_env->me_psize);
 
 						memcpy(&pg, NODEDATA(ni), sizeof(pg));
 						memcpy(NODEDATA(ni), &my->mc_next_pgno, sizeof(pgno_t));
-						rc = mdb_page_get(&mc, pg, &omp, NULL);
+						rc = MDB_PAGE_GET(&mc, pg, dpages, &omp, NULL);
 						if (rc)
 							goto done;
 						if (my->mc_wlen[toggle] >= MDB_WBUF) {
@@ -10067,8 +10075,8 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags)
 						mo->mp_pgno = my->mc_next_pgno;
 						my->mc_next_pgno += omp->mp_pages;
 						my->mc_wlen[toggle] += my->mc_env->me_psize;
-						if (omp->mp_pages > 1) {
-							my->mc_olen[toggle] = my->mc_env->me_psize * (omp->mp_pages - 1);
+						if (dpages > 1) {
+							my->mc_olen[toggle] = my->mc_env->me_psize * (dpages - 1);
 							my->mc_over[toggle] = (char *)omp + my->mc_env->me_psize;
 							rc = mdb_env_cthr_toggle(my, 1);
 							if (rc)
@@ -10103,7 +10111,7 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags)
 again:
 				ni = NODEPTR(mp, mc.mc_ki[mc.mc_top]);
 				pg = NODEPGNO(ni);
-				rc = mdb_page_get(&mc, pg, &mp, NULL);
+				rc = MDB_PAGE_GET(&mc, pg, 1, &mp, NULL);
 				if (rc)
 					goto done;
 				mc.mc_top++;
@@ -10749,7 +10757,7 @@ mdb_drop0(MDB_cursor *mc, int subs)
 		if (MDB_REMAPPING(mc->mc_txn->mt_env->me_flags)) {
 		/* bump refcount for mx's pages */
 		for (i=0; i<mc->mc_snum; i++)
-			mdb_page_get(&mx, mc->mc_pg[i]->mp_pgno, &mx.mc_pg[i], NULL);
+			MDB_PAGE_GET(&mx, mc->mc_pg[i]->mp_pgno, 1, &mx.mc_pg[i], NULL);
 		}
 
 		while (mc->mc_snum > 0) {
@@ -10762,7 +10770,8 @@ mdb_drop0(MDB_cursor *mc, int subs)
 						MDB_page *omp;
 						pgno_t pg;
 						memcpy(&pg, NODEDATA(ni), sizeof(pg));
-						rc = mdb_page_get(mc, pg, &omp, NULL);
+						/* page count is irrelevant here */
+						rc = MDB_PAGE_GET(mc, pg, 1, &omp, NULL);
 						if (rc != 0)
 							goto done;
 						mdb_cassert(mc, IS_OVERFLOW(omp));

From fbf35a163f38d7f6b76cbbe32bd5e5dabd96607a Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 1 Jul 2017 18:57:09 +0200
Subject: [PATCH 309/504] Rename mdb_env_close0() -> mdb_env_close_active()

Hopefully we'll remember now what goes in which function.
---
 libraries/liblmdb/mdb.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 8db0b80046..81d1649b8d 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1593,9 +1593,9 @@ static int  mdb_env_read_header(MDB_env *env, MDB_meta *meta);
 static MDB_meta *mdb_env_pick_meta(const MDB_env *env);
 static int  mdb_env_write_meta(MDB_txn *txn);
 #ifdef MDB_USE_POSIX_MUTEX /* Drop unused excl arg */
-# define mdb_env_close0(env, excl) mdb_env_close1(env)
+# define mdb_env_close_active(env, excl) mdb_env_close1(env)
 #endif
-static void mdb_env_close0(MDB_env *env, int excl);
+static void mdb_env_close_active(MDB_env *env, int excl);
 
 static MDB_node *mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp);
 static int  mdb_node_add(MDB_cursor *mc, indx_t indx,
@@ -5397,7 +5397,7 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 #endif
 	}
 #endif
-	flags |= MDB_ENV_ACTIVE;	/* tell mdb_env_close0() to clean up */
+	flags |= MDB_ENV_ACTIVE; /* tell mdb_env_close_active() to clean up */
 
 	if (flags & MDB_RDONLY) {
 		/* silently ignore WRITEMAP when we're only getting read access */
@@ -5504,15 +5504,15 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 
 leave:
 	if (rc) {
-		mdb_env_close0(env, excl);
+		mdb_env_close_active(env, excl);
 	}
 	mdb_fname_destroy(fname);
 	return rc;
 }
 
-/** Destroy resources from mdb_env_open(), clear our readers & DBIs */
+/** When #MDB_ENV_ACTIVE: Clear #mdb_env_open()ed resources, release readers */
 static void ESECT
-mdb_env_close0(MDB_env *env, int excl)
+mdb_env_close_active(MDB_env *env, int excl)
 {
 	int i;
 
@@ -5659,7 +5659,7 @@ mdb_env_close(MDB_env *env)
 		free(dp);
 	}
 
-	mdb_env_close0(env, 0);
+	mdb_env_close_active(env, 0);
 	free(env);
 }
 

From b89f8fc9bcaee2fc9b3b9197f6bb04e0bd31515f Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 29 Jun 2017 16:18:52 +0100
Subject: [PATCH 310/504] Page-level encryption support

Currently encrypts all but the meta pages
Still needs to store/retrieve the initialization vector
---
 libraries/liblmdb/lmdb.h |  31 ++++++
 libraries/liblmdb/mdb.c  | 223 ++++++++++++++++++++++++++++++++++++---
 libraries/liblmdb/midl.h |   4 +-
 3 files changed, 243 insertions(+), 15 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 7eb0a1ea16..c471706d8b 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -312,11 +312,28 @@ typedef int  (MDB_cmp_func)(const MDB_val *a, const MDB_val *b);
  */
 typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *relctx);
 
+#if MDB_RPAGE_CACHE
+/** @brief A callback function used to encrypt/decrypt pages in the env.
+ *
+ * Encrypt or decrypt the data in src and store the result in dst using the
+ * provided key. The result must be the same number of bytes as the input.
+ * The input size will always be a multiple of the page size.
+ * @param[in] src The input data to be transformed.
+ * @param[out] dst Storage for the result.
+ * @param[in] key An array of two values: key[0] is the encryption key,
+ * and key[1] is the initialization vector.
+ * @param[in] encdec 1 to encrypt, 0 to decrypt.
+ */
+typedef void (MDB_enc_func)(const MDB_val *src, MDB_val *dst, const MDB_val *key, int encdec);
+#endif
+
 /** @defgroup	mdb_env	Environment Flags
  *	@{
  */
 	/** mmap at a fixed address (experimental) */
 #define MDB_FIXEDMAP	0x01
+	/** encrypted DB - read-only flag, set by #mdb_env_set_encrypt() */
+#define MDB_ENCRYPT		0x2000U
 	/** no environment directory */
 #define MDB_NOSUBDIR	0x4000
 	/** don't fsync after commit */
@@ -975,6 +992,20 @@ typedef void MDB_assert_func(MDB_env *env, const char *msg);
 	 */
 int  mdb_env_set_assert(MDB_env *env, MDB_assert_func *func);
 
+#if MDB_RPAGE_CACHE
+	/** @brief Set encryption on an environment.
+	 *
+	 * This must be called before #mdb_env_open(). It implicitly sets #MDB_REMAP_CHUNK
+	 * on the env.
+	 * @param[in] env An environment handle returned by #mdb_env_create().
+	 * @param[in] func An #MDB_enc_func function.
+	 * @param[in] key An array of two values: key[0] is the encryption key,
+	 * and key[1] is the initialization vector.
+	 * @return A non-zero error value on failure and 0 on success.
+	 */
+int mdb_env_set_encrypt(MDB_env *env, MDB_enc_func *func, const MDB_val *key);
+#endif
+
 	/** @brief Create a transaction for use with the environment.
 	 *
 	 * The transaction handle may be discarded using #mdb_txn_abort() or #mdb_txn_commit().
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 81d1649b8d..f2c3557fcc 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1521,6 +1521,8 @@ struct MDB_env {
 #define MDB_ERPAGE_SIZE	16384
 #define MDB_ERPAGE_MAX	(MDB_ERPAGE_SIZE-1)
 	unsigned int me_rpcheck;
+	MDB_enc_func *me_encfunc;	/**< encrypt env data */
+	MDB_val		me_enckey[2];	/**< key and IV for env encryption */
 #endif
 	void		*me_userctx;	 /**< User-settable context */
 	MDB_assert_func *me_assert_func; /**< Callback for assertion failures */
@@ -1985,7 +1987,7 @@ mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b)
  * Set #MDB_TXN_ERROR on failure.
  */
 static MDB_page *
-mdb_page_malloc(MDB_txn *txn, unsigned num)
+mdb_page_malloc(MDB_txn *txn, unsigned num, int init)
 {
 	MDB_env *env = txn->mt_env;
 	MDB_page *ret = env->me_dpages;
@@ -2009,7 +2011,7 @@ mdb_page_malloc(MDB_txn *txn, unsigned num)
 	}
 	if ((ret = malloc(sz)) != NULL) {
 		VGMEMP_ALLOC(env, ret, sz);
-		if (!(env->me_flags & MDB_NOMEMINIT)) {
+		if (init && !(env->me_flags & MDB_NOMEMINIT)) {
 			memset((char *)ret + off, 0, psize);
 			ret->mp_pad = 0;
 		}
@@ -2043,6 +2045,24 @@ mdb_dpage_free(MDB_env *env, MDB_page *dp)
 	}
 }
 
+#if MDB_RPAGE_CACHE
+/** Free an encrypted dirty page
+ * We can't check if it's an overflow page,
+ * caller must tell us how many are being freed.
+ */
+static void
+mdb_dpage_free_n(MDB_env *env, MDB_page *dp, int num)
+{
+	if (num == 1)
+		mdb_page_free(env, dp);
+	else {
+		/* large pages just get freed directly */
+		VGMEMP_FREE(env, dp);
+		free(dp);
+	}
+}
+#endif
+
 /**	Return all dirty pages to dpage list */
 static void
 mdb_dlist_free(MDB_txn *txn)
@@ -2589,7 +2609,7 @@ search_done:
 	if (env->me_flags & MDB_WRITEMAP) {
 		np = (MDB_page *)(env->me_map + env->me_psize * pgno);
 	} else {
-		if (!(np = mdb_page_malloc(txn, num))) {
+		if (!(np = mdb_page_malloc(txn, num, 1))) {
 			rc = ENOMEM;
 			goto fail;
 		}
@@ -2669,7 +2689,7 @@ mdb_page_unspill(MDB_txn *txn, MDB_page *mp, MDB_page **ret)
 			if (env->me_flags & MDB_WRITEMAP) {
 				np = mp;
 			} else {
-				np = mdb_page_malloc(txn, num);
+				np = mdb_page_malloc(txn, num, 1);
 				if (!np)
 					return ENOMEM;
 				if (num > 1)
@@ -2757,7 +2777,7 @@ mdb_page_touch(MDB_cursor *mc)
 		}
 		mdb_cassert(mc, dl[0].mid < MDB_IDL_UM_MAX);
 		/* No - copy it */
-		np = mdb_page_malloc(txn, 1);
+		np = mdb_page_malloc(txn, 1, 1);
 		if (!np)
 			return ENOMEM;
 		mid.mid = pgno;
@@ -3372,13 +3392,23 @@ mdb_txn_end(MDB_txn *txn, unsigned mode)
 			if (tl[i].mid & (MDB_RPAGE_CHUNK-1)) {
 				/* tmp overflow pages that we didn't share in env */
 				munmap(tl[i].mptr, tl[i].mcnt * env->me_psize);
+				if (tl[i].menc) {
+					free(tl[i].menc);
+					tl[i].menc = NULL;
+				}
 			} else {
 				x = mdb_mid3l_search(el, tl[i].mid);
 				if (tl[i].mptr == el[x].mptr) {
 					el[x].mref--;
+					if (!el[x].mref)
+						el[x].muse = 0;
 				} else {
 					/* another tmp overflow page */
 					munmap(tl[i].mptr, tl[i].mcnt * env->me_psize);
+					if (tl[i].menc) {
+						free(tl[i].menc);
+						tl[i].menc = NULL;
+					}
 				}
 			}
 		}
@@ -3628,6 +3658,10 @@ mdb_page_flush(MDB_txn *txn, int keep)
 	off_t		pos = 0;
 	pgno_t		pgno = 0;
 	MDB_page	*dp = NULL;
+#if MDB_RPAGE_CACHE
+	int			nump = 1;
+	MDB_page	*encp;
+#endif
 #ifdef _WIN32
 	OVERLAPPED	ov;
 #else
@@ -3669,11 +3703,33 @@ mdb_page_flush(MDB_txn *txn, int keep)
 			dp->mp_flags &= ~P_DIRTY;
 			pos = pgno * psize;
 			size = psize;
+#if MDB_RPAGE_CACHE
+			nump = 1;
+			if (IS_OVERFLOW(dp)) {
+				nump = dp->mp_pages;
+				size *= dp->mp_pages;
+			}
+#else
 			if (IS_OVERFLOW(dp)) size *= dp->mp_pages;
+#endif
 		}
 #ifdef _WIN32
 		else break;
 
+#if MDB_RPAGE_CACHE
+		if (env->me_encfunc) {
+			MDB_val in, out;
+			encp = mdb_page_malloc(txn, nump. 0);
+			if (!encp)
+				return ENOMEM;
+			in.mv_size = size;
+			in.mv_data = dp;
+			out.mv_size = size;
+			out.mv_data = encp;
+			env->me_encfunc(&in, &out, env->me_enckey, 1);
+			dp = encp;
+		}
+#endif
 		/* Windows actually supports scatter/gather I/O, but only on
 		 * unbuffered file handles. Since we're relying on the OS page
 		 * cache for all our data, that's self-defeating. So we just
@@ -3685,8 +3741,14 @@ mdb_page_flush(MDB_txn *txn, int keep)
 		memset(&ov, 0, sizeof(ov));
 		ov.Offset = pos & 0xffffffff;
 		ov.OffsetHigh = pos >> 16 >> 16;
-		if (!WriteFile(env->me_fd, dp, size, NULL, &ov)) {
+		rc = 0;
+		if (!WriteFile(env->me_fd, dp, size, NULL, &ov))
 			rc = ErrCode();
+#if MDB_RPAGE_CACHE
+		if (env->me_encfunc)
+			mdb_dpage_free_n(env, dp, nump);
+#endif
+		if (rc) {
 			DPRINTF(("WriteFile: %d", rc));
 			return rc;
 		}
@@ -3695,6 +3757,7 @@ mdb_page_flush(MDB_txn *txn, int keep)
 		if (pos!=next_pos || n==MDB_COMMIT_PAGES || wsize+size>MAX_WRITE) {
 			if (n) {
 retry_write:
+				rc = 0;
 				/* Write previous page(s) */
 #ifdef MDB_USE_PWRITEV
 				wres = pwritev(env->me_fd, iov, n, wpos);
@@ -3708,9 +3771,11 @@ retry_seek:
 						if (rc == EINTR)
 							goto retry_seek;
 						DPRINTF(("lseek: %s", strerror(rc)));
-						return rc;
+						wres = wsize;
+					} else {
+						rc = 0;
+						wres = writev(env->me_fd, iov, n);
 					}
-					wres = writev(env->me_fd, iov, n);
 				}
 #endif
 				if (wres != wsize) {
@@ -3723,8 +3788,18 @@ retry_seek:
 						rc = EIO; /* TODO: Use which error code? */
 						DPUTS("short write, filesystem full?");
 					}
-					return rc;
 				}
+#if MDB_RPAGE_CACHE
+				if (env->me_encfunc) {
+					int j, num1;
+					for (j=0; j<n; j++) {
+						num1 = 1 + (iov[j].iov_len > psize);
+						mdb_dpage_free_n(env, (MDB_page *)iov[j].iov_base, num1);
+					}
+				}
+#endif
+				if (rc)
+					return rc;
 				n = 0;
 			}
 			if (i > pagecount)
@@ -3732,6 +3807,20 @@ retry_seek:
 			wpos = pos;
 			wsize = 0;
 		}
+#if MDB_RPAGE_CACHE
+		if (env->me_encfunc) {
+			MDB_val in, out;
+			encp = mdb_page_malloc(txn, nump, 0);
+			if (!encp)
+				return ENOMEM;
+			in.mv_size = size;
+			in.mv_data = dp;
+			out.mv_size = size;
+			out.mv_data = encp;
+			env->me_encfunc(&in, &out, env->me_enckey, 1);
+			dp = encp;
+		}
+#endif
 		DPRINTF(("committing page %"Yu, pgno));
 		next_pos = pos + size;
 		iov[n].iov_len = size;
@@ -4820,6 +4909,8 @@ mdb_env_open2(MDB_env *env)
 			return i;
 		}
 	}
+	if ((env->me_flags ^ env->me_metas[0]->mm_flags) & MDB_ENCRYPT)
+		return MDB_INCOMPATIBLE;
 
 	env->me_maxfree_1pg = (env->me_psize - PAGEHDRSZ) / sizeof(pgno_t) - 1;
 	env->me_nodemax = (((env->me_psize - PAGEHDRSZ) / MDB_MINKEYS) & -2)
@@ -5660,6 +5751,9 @@ mdb_env_close(MDB_env *env)
 	}
 
 	mdb_env_close_active(env, 0);
+#if MDB_RPAGE_CACHE
+	free(env->me_enckey[0].mv_data);
+#endif
 	free(env);
 }
 
@@ -5909,6 +6003,8 @@ mdb_cursor_push(MDB_cursor *mc, MDB_page *mp)
 }
 
 #if MDB_RPAGE_CACHE
+static void mdb_rpage_decrypt(MDB_env *env, MDB_ID3 *id3, int rem, int numpgs);
+
 /** Map a read-only page.
  * There are two levels of tracking in use, a per-txn list and a per-env list.
  * ref'ing and unref'ing the per-txn list is faster since it requires no
@@ -5966,6 +6062,7 @@ mdb_rpage_get(MDB_txn *txn, pgno_t pg0, int numpgs, MDB_page **ret)
 	MDB_ID3L tl = txn->mt_rpages;
 	MDB_ID3L el = env->me_rpages;
 	MDB_ID3 id3;
+	char *base;
 	unsigned x, rem;
 	pgno_t pgno;
 	int rc, retries = 1;
@@ -5994,6 +6091,8 @@ mdb_rpage_get(MDB_txn *txn, pgno_t pg0, int numpgs, MDB_page **ret)
 	pgno = pg0 ^ rem;
 
 	id3.mid = 0;
+	id3.menc = NULL;
+	id3.muse = 0;
 	x = mdb_mid3l_search(tl, pgno);
 	if (x <= tl[0].mid && tl[x].mid == pgno) {
 		if (x != tl[0].mid && tl[x+1].mid == pg0)
@@ -6006,6 +6105,12 @@ mdb_rpage_get(MDB_txn *txn, pgno_t pg0, int numpgs, MDB_page **ret)
 			MAP(rc, env, id3.mptr, len, off);
 			if (rc)
 				return rc;
+			/* setup for encryption */
+			if (env->me_encfunc) {
+				id3.menc = malloc(len);
+				if (!id3.menc)
+					return ENOMEM;
+			}
 			/* check for local-only page */
 			if (rem) {
 				mdb_tassert(txn, tl[x].mid != pg0);
@@ -6019,6 +6124,10 @@ mdb_rpage_get(MDB_txn *txn, pgno_t pg0, int numpgs, MDB_page **ret)
 				/* ignore the mapping we got from env, use new one */
 				tl[x].mptr = id3.mptr;
 				tl[x].mcnt = id3.mcnt;
+				if (tl[x].menc)
+					free(tl[x].menc);
+				tl[x].menc = id3.menc;
+				tl[x].muse = id3.muse;
 				/* if no active ref, see if we can replace in env */
 				if (!tl[x].mref) {
 					unsigned i;
@@ -6029,6 +6138,10 @@ mdb_rpage_get(MDB_txn *txn, pgno_t pg0, int numpgs, MDB_page **ret)
 						munmap(el[i].mptr, el[i].mcnt * env->me_psize);
 						el[i].mptr = tl[x].mptr;
 						el[i].mcnt = tl[x].mcnt;
+						if (el[i].menc)
+							free(el[i].menc);
+						el[i].menc = tl[x].menc;
+						el[i].muse = tl[x].muse;
 					} else {
 						/* there are others, remove ourself */
 						el[i].mref--;
@@ -6039,7 +6152,13 @@ mdb_rpage_get(MDB_txn *txn, pgno_t pg0, int numpgs, MDB_page **ret)
 		}
 		id3.mptr = tl[x].mptr;
 		id3.mcnt = tl[x].mcnt;
+		id3.menc = tl[x].menc;
+		id3.muse = tl[x].muse;
 		tl[x].mref++;
+		if (env->me_encfunc) {
+			mdb_rpage_decrypt(env, &id3, rem, numpgs);
+			tl[x].muse = id3.muse;
+		}
 		goto ok;
 	}
 
@@ -6056,6 +6175,8 @@ retry:
 				/* tmp overflow pages don't go to env */
 				if (tl[i].mid & (MDB_RPAGE_CHUNK-1)) {
 					munmap(tl[i].mptr, tl[i].mcnt * env->me_psize);
+					if (tl[i].menc)
+						free(tl[i].menc);
 					continue;
 				}
 				x = mdb_mid3l_search(el, tl[i].mid);
@@ -6103,23 +6224,45 @@ retry:
 		x = mdb_mid3l_search(el, pgno);
 		if (x <= el[0].mid && el[x].mid == pgno) {
 			id3.mptr = el[x].mptr;
+			id3.menc = el[x].menc;
+			id3.muse = el[x].muse;
 			/* check for overflow size */
 			if (id3.mcnt > el[x].mcnt) {
 				SET_OFF(off, pgno * env->me_psize);
 				MAP(rc, env, id3.mptr, len, off);
 				if (rc)
 					goto fail;
+				if (env->me_encfunc) {
+					id3.menc = malloc(len);
+					if (!id3.menc) {
+						rc = ENOMEM;
+						goto fail;
+					}
+					id3.muse = 0;
+				}
 				if (!el[x].mref) {
 					munmap(el[x].mptr, env->me_psize * el[x].mcnt);
 					el[x].mptr = id3.mptr;
 					el[x].mcnt = id3.mcnt;
+					if (el[x].menc)
+						free(el[x].menc);
+					el[x].menc = id3.menc;
+					el[x].muse = id3.muse;
 				} else {
 					id3.mid = pg0;
+					if (env->me_encfunc) {
+						mdb_rpage_decrypt(env, &id3, rem, numpgs);
+						el[x].muse = id3.muse;
+					}
 					pthread_mutex_unlock(&env->me_rpmutex);
 					goto found;
 				}
 			}
 			el[x].mref++;
+			if (env->me_encfunc) {
+				mdb_rpage_decrypt(env, &id3, rem, numpgs);
+				el[x].muse = id3.muse;
+			}
 			pthread_mutex_unlock(&env->me_rpmutex);
 			goto found;
 		}
@@ -6130,6 +6273,8 @@ retry:
 				if (!el[i].mref) {
 					if (!y) y = i;
 					munmap(el[i].mptr, env->me_psize * el[i].mcnt);
+					if (el[i].menc)
+						free(el[i].menc);
 				}
 			}
 			if (!y) {
@@ -6162,6 +6307,14 @@ fail:
 			pthread_mutex_unlock(&env->me_rpmutex);
 			return rc;
 		}
+		if (env->me_encfunc) {
+			id3.menc = malloc(len);
+			if (!id3.menc) {
+				rc = ENOMEM;
+				goto fail;
+			}
+			mdb_rpage_decrypt(env, &id3, rem, numpgs);
+		}
 		mdb_mid3l_insert(el, &id3);
 		pthread_mutex_unlock(&env->me_rpmutex);
 found:
@@ -6170,7 +6323,10 @@ found:
 		return MDB_TXN_FULL;
 	}
 ok:
-	p = (MDB_page *)((char *)id3.mptr + rem * env->me_psize);
+	base = (char *)(env->me_encfunc ? id3.menc : id3.mptr);
+	p = (MDB_page *)(base + rem * env->me_psize);
+	if (env->me_encfunc)
+		mdb_rpage_decrypt(env, &id3, rem, numpgs);
 #if MDB_DEBUG	/* we don't need this check any more */
 	if (IS_OVERFLOW(p)) {
 		mdb_tassert(txn, p->mp_pages + rem <= id3.mcnt);
@@ -6179,6 +6335,19 @@ ok:
 	*ret = p;
 	return MDB_SUCCESS;
 }
+
+static void mdb_rpage_decrypt(MDB_env *env, MDB_ID3 *id3, int rem, int numpgs)
+{
+	if (!(id3->muse & (1 << rem))) {
+		MDB_val in, out;
+		id3->muse |= (1 << rem);
+		in.mv_size = numpgs * env->me_psize;
+		in.mv_data = (char *)id3->mptr + rem * env->me_psize;
+		out.mv_size = in.mv_size;
+		out.mv_data = (char *)id3->menc + rem * env->me_psize;
+		env->me_encfunc(&in, &out, env->me_enckey, 0);
+	}
+}
 #endif
 
 /** Find the address of the page corresponding to a given page number.
@@ -6557,15 +6726,14 @@ mdb_node_read(MDB_cursor *mc, MDB_node *leaf, MDB_val *data)
 		MDB_PAGE_UNREF(mc->mc_txn, MC_OVPG(mc));
 		MC_SET_OVPG(mc, NULL);
 	}
+	data->mv_size = NODEDSZ(leaf);
 	if (!F_ISSET(leaf->mn_flags, F_BIGDATA)) {
-		data->mv_size = NODEDSZ(leaf);
 		data->mv_data = NODEDATA(leaf);
 		return MDB_SUCCESS;
 	}
 
 	/* Read overflow data.
 	 */
-	data->mv_size = NODEDSZ(leaf);
 	memcpy(&pgno, NODEDATA(leaf), sizeof(pgno));
 	{
 #if MDB_RPAGE_CACHE
@@ -7702,7 +7870,7 @@ current:
 				if (level > 1) {
 					/* It is writable only in a parent txn */
 					size_t sz = (size_t) env->me_psize * ovpages, off;
-					MDB_page *np = mdb_page_malloc(mc->mc_txn, ovpages);
+					MDB_page *np = mdb_page_malloc(mc->mc_txn, ovpages, 1);
 					MDB_ID2 id2;
 					if (!np)
 						return ENOMEM;
@@ -9556,7 +9724,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 			nsize = EVEN(nsize);
 
 			/* grab a page to hold a temporary copy */
-			copy = mdb_page_malloc(mc->mc_txn, 1);
+			copy = mdb_page_malloc(mc->mc_txn, 1, 1);
 			if (copy == NULL) {
 				rc = ENOMEM;
 				goto done;
@@ -10474,6 +10642,33 @@ mdb_env_set_assert(MDB_env *env, MDB_assert_func *func)
 	return MDB_SUCCESS;
 }
 
+#if MDB_RPAGE_CACHE
+int ESECT
+mdb_env_set_encrypt(MDB_env *env, MDB_enc_func *func, const MDB_val *key)
+{
+	char *kdata, *ivdata;
+
+	if (!env || !func || !key)
+		return EINVAL;
+	if (env->me_flags & MDB_ENV_ACTIVE)
+		return EINVAL;
+	if (! (kdata = malloc(key[0].mv_size + key[1].mv_size)))
+		return ENOMEM;
+
+	ivdata = kdata + key[0].mv_size;
+	memcpy(kdata,  key[0].mv_data, key[0].mv_size);
+	memcpy(ivdata, key[1].mv_data, key[1].mv_size);
+	free(env->me_enckey[0].mv_data);
+	env->me_enckey[0].mv_data = kdata;
+	env->me_enckey[0].mv_size = key[0].mv_size;
+	env->me_enckey[1].mv_data = ivdata;
+	env->me_enckey[1].mv_size = key[1].mv_size;
+	env->me_encfunc = func;
+	env->me_flags |= MDB_REMAP_CHUNKS | MDB_ENCRYPT;
+	return MDB_SUCCESS;
+}
+#endif
+
 int ESECT
 mdb_env_get_path(MDB_env *env, const char **arg)
 {
diff --git a/libraries/liblmdb/midl.h b/libraries/liblmdb/midl.h
index ac6bdd0ecb..1a603da332 100644
--- a/libraries/liblmdb/midl.h
+++ b/libraries/liblmdb/midl.h
@@ -182,8 +182,10 @@ int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id );
 typedef struct MDB_ID3 {
 	MDB_ID mid;		/**< The ID */
 	void *mptr;		/**< The pointer */
+	void *menc;		/**< Decrypted pointer */
 	unsigned int mcnt;		/**< Number of pages */
-	unsigned int mref;		/**< Refcounter */
+	unsigned short mref;	/**< Refcounter */
+	unsigned short muse;	/**< Bitmap of used pages */
 } MDB_ID3;
 
 typedef MDB_ID3 *MDB_ID3L;

From 525a2cce9131fd9c2bdef6662c99ab6a033ceb74 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 29 Jun 2017 18:22:34 +0100
Subject: [PATCH 311/504] Add test code for remap / encryption

---
 libraries/liblmdb/Makefile      |   8 +-
 libraries/liblmdb/chacha8.c     | 183 ++++++++++++++++++++++++++++++
 libraries/liblmdb/chacha8.h     |   8 ++
 libraries/liblmdb/mdb.c         |   5 +
 libraries/liblmdb/mtest_enc.c   | 192 ++++++++++++++++++++++++++++++++
 libraries/liblmdb/mtest_remap.c | 177 +++++++++++++++++++++++++++++
 6 files changed, 572 insertions(+), 1 deletion(-)
 create mode 100644 libraries/liblmdb/chacha8.c
 create mode 100644 libraries/liblmdb/chacha8.h
 create mode 100644 libraries/liblmdb/mtest_enc.c
 create mode 100644 libraries/liblmdb/mtest_remap.c

diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile
index 72d0984607..2ff32345ce 100644
--- a/libraries/liblmdb/Makefile
+++ b/libraries/liblmdb/Makefile
@@ -42,7 +42,11 @@ ILIBS	= liblmdb.a liblmdb$(SOEXT)
 IPROGS	= mdb_stat mdb_copy mdb_dump mdb_load
 IDOCS	= mdb_stat.1 mdb_copy.1 mdb_dump.1 mdb_load.1
 PROGS	= $(IPROGS) mtest mtest2 mtest3 mtest4 mtest5
+RPROGS	= mtest_remap mtest_enc
+
 all:	$(ILIBS) $(PROGS)
+# Requires CPPFLAGS=-DMDB_VL32 and/or -DMDB_RPAGE_CACHE
+rall:	all $(RPROGS)
 
 install: $(ILIBS) $(IPROGS) $(IHDRS)
 	mkdir -p $(DESTDIR)$(bindir)
@@ -55,7 +59,7 @@ install: $(ILIBS) $(IPROGS) $(IHDRS)
 	for f in $(IDOCS); do cp $$f $(DESTDIR)$(mandir)/man1; done
 
 clean:
-	rm -rf $(PROGS) *.[ao] *.[ls]o *~ testdb
+	rm -rf $(PROGS) $(RPROGS) *.[ao] *.[ls]o *~ testdb
 
 test:	all
 	rm -rf testdb && mkdir testdb
@@ -78,6 +82,8 @@ mtest3:	mtest3.o liblmdb.a
 mtest4:	mtest4.o liblmdb.a
 mtest5:	mtest5.o liblmdb.a
 mtest6:	mtest6.o liblmdb.a
+mtest_remap:  mtest_remap.o liblmdb.a
+mtest_enc:    mtest_enc.o chacha8.o liblmdb.a
 
 mdb.o: mdb.c lmdb.h midl.h
 	$(CC) $(CFLAGS) $(CPPFLAGS) -c mdb.c
diff --git a/libraries/liblmdb/chacha8.c b/libraries/liblmdb/chacha8.c
new file mode 100644
index 0000000000..87201d0f50
--- /dev/null
+++ b/libraries/liblmdb/chacha8.c
@@ -0,0 +1,183 @@
+/*
+chacha-merged.c version 20080118
+D. J. Bernstein
+Public domain.
+*/
+
+#include <memory.h>
+#include <stdio.h>
+#include <sys/param.h>
+
+#include "chacha8.h"
+#if 0
+#include "common/int-util.h"
+#include "warnings.h"
+#endif
+
+#if BYTE_ORDER == LITTLE_ENDIAN
+#define SWAP32LE(x)	(x)
+#else
+#define SWAP32LE(x)  ((((uint32_t) (x) & 0x000000ff) << 24) | \
+  (((uint32_t) (x) & 0x0000ff00) <<  8) | \
+  (((uint32_t) (x) & 0x00ff0000) >>  8) | \
+  (((uint32_t) (x) & 0xff000000) >> 24))
+#endif
+
+/*
+ * The following macros are used to obtain exact-width results.
+ */
+#define U8V(v) ((uint8_t)(v) & UINT8_C(0xFF))
+#define U32V(v) ((uint32_t)(v) & UINT32_C(0xFFFFFFFF))
+
+/*
+ * The following macros load words from an array of bytes with
+ * different types of endianness, and vice versa.
+ */
+#define U8TO32_LITTLE(p) SWAP32LE(((uint32_t*)(p))[0])
+#define U32TO8_LITTLE(p, v) (((uint32_t*)(p))[0] = SWAP32LE(v))
+
+#define ROTATE(v,c) (rol32(v,c))
+#define XOR(v,w) ((v) ^ (w))
+#define PLUS(v,w) (U32V((v) + (w)))
+#define PLUSONE(v) (PLUS((v),1))
+
+#define QUARTERROUND(a,b,c,d) \
+  a = PLUS(a,b); d = ROTATE(XOR(d,a),16); \
+  c = PLUS(c,d); b = ROTATE(XOR(b,c),12); \
+  a = PLUS(a,b); d = ROTATE(XOR(d,a), 8); \
+  c = PLUS(c,d); b = ROTATE(XOR(b,c), 7);
+
+static const char sigma[] = "expand 32-byte k";
+
+static uint32_t rol32(uint32_t x, int r) {
+	return (x << (r & 31)) | (x >> (-r & 31));
+}
+
+void chacha8(const void* data, size_t length, const uint8_t* key, const uint8_t* iv, char* cipher) {
+  uint32_t x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15;
+  uint32_t j0, j1, j2, j3, j4, j5, j6, j7, j8, j9, j10, j11, j12, j13, j14, j15;
+  char* ctarget = 0;
+  char tmp[64];
+  int i;
+
+  if (!length) return;
+
+  j0  = U8TO32_LITTLE(sigma + 0);
+  j1  = U8TO32_LITTLE(sigma + 4);
+  j2  = U8TO32_LITTLE(sigma + 8);
+  j3  = U8TO32_LITTLE(sigma + 12);
+  j4  = U8TO32_LITTLE(key + 0);
+  j5  = U8TO32_LITTLE(key + 4);
+  j6  = U8TO32_LITTLE(key + 8);
+  j7  = U8TO32_LITTLE(key + 12);
+  j8  = U8TO32_LITTLE(key + 16);
+  j9  = U8TO32_LITTLE(key + 20);
+  j10 = U8TO32_LITTLE(key + 24);
+  j11 = U8TO32_LITTLE(key + 28);
+  j12 = 0;
+  j13 = 0;
+  j14 = U8TO32_LITTLE(iv + 0);
+  j15 = U8TO32_LITTLE(iv + 4);
+
+  for (;;) {
+    if (length < 64) {
+      memcpy(tmp, data, length);
+      data = tmp;
+      ctarget = cipher;
+      cipher = tmp;
+    }
+    x0  = j0;
+    x1  = j1;
+    x2  = j2;
+    x3  = j3;
+    x4  = j4;
+    x5  = j5;
+    x6  = j6;
+    x7  = j7;
+    x8  = j8;
+    x9  = j9;
+    x10 = j10;
+    x11 = j11;
+    x12 = j12;
+    x13 = j13;
+    x14 = j14;
+    x15 = j15;
+    for (i = 8;i > 0;i -= 2) {
+      QUARTERROUND( x0, x4, x8,x12)
+      QUARTERROUND( x1, x5, x9,x13)
+      QUARTERROUND( x2, x6,x10,x14)
+      QUARTERROUND( x3, x7,x11,x15)
+      QUARTERROUND( x0, x5,x10,x15)
+      QUARTERROUND( x1, x6,x11,x12)
+      QUARTERROUND( x2, x7, x8,x13)
+      QUARTERROUND( x3, x4, x9,x14)
+    }
+    x0  = PLUS( x0, j0);
+    x1  = PLUS( x1, j1);
+    x2  = PLUS( x2, j2);
+    x3  = PLUS( x3, j3);
+    x4  = PLUS( x4, j4);
+    x5  = PLUS( x5, j5);
+    x6  = PLUS( x6, j6);
+    x7  = PLUS( x7, j7);
+    x8  = PLUS( x8, j8);
+    x9  = PLUS( x9, j9);
+    x10 = PLUS(x10,j10);
+    x11 = PLUS(x11,j11);
+    x12 = PLUS(x12,j12);
+    x13 = PLUS(x13,j13);
+    x14 = PLUS(x14,j14);
+    x15 = PLUS(x15,j15);
+
+    x0  = XOR( x0,U8TO32_LITTLE((uint8_t*)data +  0));
+    x1  = XOR( x1,U8TO32_LITTLE((uint8_t*)data +  4));
+    x2  = XOR( x2,U8TO32_LITTLE((uint8_t*)data +  8));
+    x3  = XOR( x3,U8TO32_LITTLE((uint8_t*)data + 12));
+    x4  = XOR( x4,U8TO32_LITTLE((uint8_t*)data + 16));
+    x5  = XOR( x5,U8TO32_LITTLE((uint8_t*)data + 20));
+    x6  = XOR( x6,U8TO32_LITTLE((uint8_t*)data + 24));
+    x7  = XOR( x7,U8TO32_LITTLE((uint8_t*)data + 28));
+    x8  = XOR( x8,U8TO32_LITTLE((uint8_t*)data + 32));
+    x9  = XOR( x9,U8TO32_LITTLE((uint8_t*)data + 36));
+    x10 = XOR(x10,U8TO32_LITTLE((uint8_t*)data + 40));
+    x11 = XOR(x11,U8TO32_LITTLE((uint8_t*)data + 44));
+    x12 = XOR(x12,U8TO32_LITTLE((uint8_t*)data + 48));
+    x13 = XOR(x13,U8TO32_LITTLE((uint8_t*)data + 52));
+    x14 = XOR(x14,U8TO32_LITTLE((uint8_t*)data + 56));
+    x15 = XOR(x15,U8TO32_LITTLE((uint8_t*)data + 60));
+
+    j12 = PLUSONE(j12);
+    if (!j12)
+    {
+      j13 = PLUSONE(j13);
+      /* stopping at 2^70 bytes per iv is user's responsibility */
+    }
+
+    U32TO8_LITTLE(cipher +  0,x0);
+    U32TO8_LITTLE(cipher +  4,x1);
+    U32TO8_LITTLE(cipher +  8,x2);
+    U32TO8_LITTLE(cipher + 12,x3);
+    U32TO8_LITTLE(cipher + 16,x4);
+    U32TO8_LITTLE(cipher + 20,x5);
+    U32TO8_LITTLE(cipher + 24,x6);
+    U32TO8_LITTLE(cipher + 28,x7);
+    U32TO8_LITTLE(cipher + 32,x8);
+    U32TO8_LITTLE(cipher + 36,x9);
+    U32TO8_LITTLE(cipher + 40,x10);
+    U32TO8_LITTLE(cipher + 44,x11);
+    U32TO8_LITTLE(cipher + 48,x12);
+    U32TO8_LITTLE(cipher + 52,x13);
+    U32TO8_LITTLE(cipher + 56,x14);
+    U32TO8_LITTLE(cipher + 60,x15);
+
+    if (length <= 64) {
+      if (length < 64) {
+        memcpy(ctarget, cipher, length);
+      }
+      return;
+    }
+    length -= 64;
+    cipher += 64;
+    data = (uint8_t*)data + 64;
+  }
+}
diff --git a/libraries/liblmdb/chacha8.h b/libraries/liblmdb/chacha8.h
new file mode 100644
index 0000000000..fed3f7b6b1
--- /dev/null
+++ b/libraries/liblmdb/chacha8.h
@@ -0,0 +1,8 @@
+#include <stdint.h>
+#include <stddef.h>
+
+#define CHACHA8_KEY_SIZE 32
+#define CHACHA8_IV_SIZE 8
+
+void chacha8(const void* data, size_t length, const uint8_t* key, const uint8_t* iv, char* cipher);
+
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index f2c3557fcc..6c7d8db456 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5464,9 +5464,14 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 	if (MDB_REMAPPING(flags)) {
 		/* silently ignore WRITEMAP with REMAP_CHUNKS */
 		flags &= ~MDB_WRITEMAP;
+#if (MDB_RPAGE_CACHE) & 2
+		/* TEST: silently ignore FIXEDMAP, so mtest*.c will work */
+		flags &= ~MDB_FIXEDMAP;
+#else
 		/* cannot support FIXEDMAP */
 		if (flags & MDB_FIXEDMAP)
 			return EINVAL;
+#endif
 	}
 
 	rc = mdb_fname_init(path, flags, &fname);
diff --git a/libraries/liblmdb/mtest_enc.c b/libraries/liblmdb/mtest_enc.c
new file mode 100644
index 0000000000..ecb0875730
--- /dev/null
+++ b/libraries/liblmdb/mtest_enc.c
@@ -0,0 +1,192 @@
+/* mtest_enc.c - memory-mapped database tester/toy with encryption */
+/*
+ * Copyright 2011-2017 Howard Chu, Symas Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the Symas
+ * Dual-Use License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * source distribution.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+#include "lmdb.h"
+#include "chacha8.h"
+
+#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr)
+#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0))
+#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \
+	"%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort()))
+
+static void encfunc(const MDB_val *src, MDB_val *dst, const MDB_val *key, int encdec)
+{
+	chacha8(src->mv_data, src->mv_size, key[0].mv_data, key[1].mv_data, dst->mv_data);
+}
+
+int main(int argc,char * argv[])
+{
+	int i = 0, j = 0, rc;
+	MDB_env *env;
+	MDB_dbi dbi;
+	MDB_val key, data;
+	MDB_txn *txn;
+	MDB_stat mst;
+	MDB_cursor *cursor, *cur2;
+	MDB_cursor_op op;
+	MDB_val enckey[2];
+	int count;
+	int *values;
+	char sval[32] = "";
+	char eiv[] = {3, 1, 4, 1, 5, 9, 2, 6};
+	char ekey[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
+		17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32};
+
+	srand(time(NULL));
+
+	    count = (rand()%384) + 64;
+	    values = (int *)malloc(count*sizeof(int));
+
+	    for(i = 0;i<count;i++) {
+			values[i] = rand()%1024;
+	    }
+    
+		enckey[0].mv_data = ekey;
+		enckey[0].mv_size = sizeof(ekey);
+		enckey[1].mv_data = eiv;
+		enckey[1].mv_size = sizeof(eiv);
+
+		E(mdb_env_create(&env));
+		E(mdb_env_set_maxreaders(env, 1));
+		E(mdb_env_set_mapsize(env, 10485760));
+		E(mdb_env_set_encrypt(env, encfunc, enckey));
+		E(mdb_env_open(env, "./testdb", 0 /*|MDB_NOSYNC*/, 0664));
+
+		E(mdb_txn_begin(env, NULL, 0, &txn));
+		E(mdb_dbi_open(txn, NULL, 0, &dbi));
+   
+		key.mv_size = sizeof(int);
+		key.mv_data = sval;
+
+		printf("Adding %d values\n", count);
+	    for (i=0;i<count;i++) {	
+			sprintf(sval, "%03x %d foo bar", values[i], values[i]);
+			/* Set <data> in each iteration, since MDB_NOOVERWRITE may modify it */
+			data.mv_size = sizeof(sval);
+			data.mv_data = sval;
+			if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) {
+				j++;
+				data.mv_size = sizeof(sval);
+				data.mv_data = sval;
+			}
+	    }
+		if (j) printf("%d duplicates skipped\n", j);
+		E(mdb_txn_commit(txn));
+		E(mdb_env_stat(env, &mst));
+
+		E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
+		E(mdb_cursor_open(txn, dbi, &cursor));
+		while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
+			printf("key: %p %.*s, data: %p %.*s\n",
+				key.mv_data,  (int) key.mv_size,  (char *) key.mv_data,
+				data.mv_data, (int) data.mv_size, (char *) data.mv_data);
+		}
+		CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
+		mdb_cursor_close(cursor);
+		mdb_txn_abort(txn);
+
+		j=0;
+		key.mv_data = sval;
+	    for (i= count - 1; i > -1; i-= (rand()%5)) {
+			j++;
+			txn=NULL;
+			E(mdb_txn_begin(env, NULL, 0, &txn));
+			sprintf(sval, "%03x ", values[i]);
+			if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) {
+				j--;
+				mdb_txn_abort(txn);
+			} else {
+				E(mdb_txn_commit(txn));
+			}
+	    }
+	    free(values);
+		printf("Deleted %d values\n", j);
+
+		E(mdb_env_stat(env, &mst));
+		E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
+		E(mdb_cursor_open(txn, dbi, &cursor));
+		printf("Cursor next\n");
+		while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
+			printf("key: %.*s, data: %.*s\n",
+				(int) key.mv_size,  (char *) key.mv_data,
+				(int) data.mv_size, (char *) data.mv_data);
+		}
+		CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
+		printf("Cursor last\n");
+		E(mdb_cursor_get(cursor, &key, &data, MDB_LAST));
+		printf("key: %.*s, data: %.*s\n",
+			(int) key.mv_size,  (char *) key.mv_data,
+			(int) data.mv_size, (char *) data.mv_data);
+		printf("Cursor prev\n");
+		while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
+			printf("key: %.*s, data: %.*s\n",
+				(int) key.mv_size,  (char *) key.mv_data,
+				(int) data.mv_size, (char *) data.mv_data);
+		}
+		CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
+		printf("Cursor last/prev\n");
+		E(mdb_cursor_get(cursor, &key, &data, MDB_LAST));
+			printf("key: %.*s, data: %.*s\n",
+				(int) key.mv_size,  (char *) key.mv_data,
+				(int) data.mv_size, (char *) data.mv_data);
+		E(mdb_cursor_get(cursor, &key, &data, MDB_PREV));
+			printf("key: %.*s, data: %.*s\n",
+				(int) key.mv_size,  (char *) key.mv_data,
+				(int) data.mv_size, (char *) data.mv_data);
+
+		mdb_cursor_close(cursor);
+		mdb_txn_abort(txn);
+
+		printf("Deleting with cursor\n");
+		E(mdb_txn_begin(env, NULL, 0, &txn));
+		E(mdb_cursor_open(txn, dbi, &cur2));
+		for (i=0; i<50; i++) {
+			if (RES(MDB_NOTFOUND, mdb_cursor_get(cur2, &key, &data, MDB_NEXT)))
+				break;
+			printf("key: %p %.*s, data: %p %.*s\n",
+				key.mv_data,  (int) key.mv_size,  (char *) key.mv_data,
+				data.mv_data, (int) data.mv_size, (char *) data.mv_data);
+			E(mdb_del(txn, dbi, &key, NULL));
+		}
+
+		printf("Restarting cursor in txn\n");
+		for (op=MDB_FIRST, i=0; i<=32; op=MDB_NEXT, i++) {
+			if (RES(MDB_NOTFOUND, mdb_cursor_get(cur2, &key, &data, op)))
+				break;
+			printf("key: %p %.*s, data: %p %.*s\n",
+				key.mv_data,  (int) key.mv_size,  (char *) key.mv_data,
+				data.mv_data, (int) data.mv_size, (char *) data.mv_data);
+		}
+		mdb_cursor_close(cur2);
+		E(mdb_txn_commit(txn));
+
+		printf("Restarting cursor outside txn\n");
+		E(mdb_txn_begin(env, NULL, 0, &txn));
+		E(mdb_cursor_open(txn, dbi, &cursor));
+		for (op=MDB_FIRST, i=0; i<=32; op=MDB_NEXT, i++) {
+			if (RES(MDB_NOTFOUND, mdb_cursor_get(cursor, &key, &data, op)))
+				break;
+			printf("key: %p %.*s, data: %p %.*s\n",
+				key.mv_data,  (int) key.mv_size,  (char *) key.mv_data,
+				data.mv_data, (int) data.mv_size, (char *) data.mv_data);
+		}
+		mdb_cursor_close(cursor);
+		mdb_txn_abort(txn);
+
+		mdb_dbi_close(env, dbi);
+		mdb_env_close(env);
+
+	return 0;
+}
diff --git a/libraries/liblmdb/mtest_remap.c b/libraries/liblmdb/mtest_remap.c
new file mode 100644
index 0000000000..2e2b12c40f
--- /dev/null
+++ b/libraries/liblmdb/mtest_remap.c
@@ -0,0 +1,177 @@
+/* mtest_remap.c - memory-mapped database tester/toy with page remapping */
+/*
+ * Copyright 2011-2017 Howard Chu, Symas Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the Symas
+ * Dual-Use License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * source distribution.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+#include "lmdb.h"
+#include "chacha8.h"
+
+#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr)
+#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0))
+#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \
+	"%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort()))
+
+int main(int argc,char * argv[])
+{
+	int i = 0, j = 0, rc;
+	MDB_env *env;
+	MDB_dbi dbi;
+	MDB_val key, data;
+	MDB_txn *txn;
+	MDB_stat mst;
+	MDB_cursor *cursor, *cur2;
+	MDB_cursor_op op;
+	int count;
+	int *values;
+	char sval[32] = "";
+
+	srand(time(NULL));
+
+	    count = (rand()%384) + 64;
+	    values = (int *)malloc(count*sizeof(int));
+
+	    for(i = 0;i<count;i++) {
+			values[i] = rand()%1024;
+	    }
+    
+		E(mdb_env_create(&env));
+		E(mdb_env_set_maxreaders(env, 1));
+		E(mdb_env_set_mapsize(env, 10485760));
+		E(mdb_env_open(env, "./testdb", MDB_REMAP_CHUNKS /*|MDB_NOSYNC*/, 0664));
+
+		E(mdb_txn_begin(env, NULL, 0, &txn));
+		E(mdb_dbi_open(txn, NULL, 0, &dbi));
+   
+		key.mv_size = sizeof(int);
+		key.mv_data = sval;
+
+		printf("Adding %d values\n", count);
+	    for (i=0;i<count;i++) {	
+			sprintf(sval, "%03x %d foo bar", values[i], values[i]);
+			/* Set <data> in each iteration, since MDB_NOOVERWRITE may modify it */
+			data.mv_size = sizeof(sval);
+			data.mv_data = sval;
+			if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) {
+				j++;
+				data.mv_size = sizeof(sval);
+				data.mv_data = sval;
+			}
+	    }
+		if (j) printf("%d duplicates skipped\n", j);
+		E(mdb_txn_commit(txn));
+		E(mdb_env_stat(env, &mst));
+
+		E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
+		E(mdb_cursor_open(txn, dbi, &cursor));
+		while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
+			printf("key: %p %.*s, data: %p %.*s\n",
+				key.mv_data,  (int) key.mv_size,  (char *) key.mv_data,
+				data.mv_data, (int) data.mv_size, (char *) data.mv_data);
+		}
+		CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
+		mdb_cursor_close(cursor);
+		mdb_txn_abort(txn);
+
+		j=0;
+		key.mv_data = sval;
+	    for (i= count - 1; i > -1; i-= (rand()%5)) {
+			j++;
+			txn=NULL;
+			E(mdb_txn_begin(env, NULL, 0, &txn));
+			sprintf(sval, "%03x ", values[i]);
+			if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) {
+				j--;
+				mdb_txn_abort(txn);
+			} else {
+				E(mdb_txn_commit(txn));
+			}
+	    }
+	    free(values);
+		printf("Deleted %d values\n", j);
+
+		E(mdb_env_stat(env, &mst));
+		E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
+		E(mdb_cursor_open(txn, dbi, &cursor));
+		printf("Cursor next\n");
+		while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
+			printf("key: %.*s, data: %.*s\n",
+				(int) key.mv_size,  (char *) key.mv_data,
+				(int) data.mv_size, (char *) data.mv_data);
+		}
+		CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
+		printf("Cursor last\n");
+		E(mdb_cursor_get(cursor, &key, &data, MDB_LAST));
+		printf("key: %.*s, data: %.*s\n",
+			(int) key.mv_size,  (char *) key.mv_data,
+			(int) data.mv_size, (char *) data.mv_data);
+		printf("Cursor prev\n");
+		while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
+			printf("key: %.*s, data: %.*s\n",
+				(int) key.mv_size,  (char *) key.mv_data,
+				(int) data.mv_size, (char *) data.mv_data);
+		}
+		CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
+		printf("Cursor last/prev\n");
+		E(mdb_cursor_get(cursor, &key, &data, MDB_LAST));
+			printf("key: %.*s, data: %.*s\n",
+				(int) key.mv_size,  (char *) key.mv_data,
+				(int) data.mv_size, (char *) data.mv_data);
+		E(mdb_cursor_get(cursor, &key, &data, MDB_PREV));
+			printf("key: %.*s, data: %.*s\n",
+				(int) key.mv_size,  (char *) key.mv_data,
+				(int) data.mv_size, (char *) data.mv_data);
+
+		mdb_cursor_close(cursor);
+		mdb_txn_abort(txn);
+
+		printf("Deleting with cursor\n");
+		E(mdb_txn_begin(env, NULL, 0, &txn));
+		E(mdb_cursor_open(txn, dbi, &cur2));
+		for (i=0; i<50; i++) {
+			if (RES(MDB_NOTFOUND, mdb_cursor_get(cur2, &key, &data, MDB_NEXT)))
+				break;
+			printf("key: %p %.*s, data: %p %.*s\n",
+				key.mv_data,  (int) key.mv_size,  (char *) key.mv_data,
+				data.mv_data, (int) data.mv_size, (char *) data.mv_data);
+			E(mdb_del(txn, dbi, &key, NULL));
+		}
+
+		printf("Restarting cursor in txn\n");
+		for (op=MDB_FIRST, i=0; i<=32; op=MDB_NEXT, i++) {
+			if (RES(MDB_NOTFOUND, mdb_cursor_get(cur2, &key, &data, op)))
+				break;
+			printf("key: %p %.*s, data: %p %.*s\n",
+				key.mv_data,  (int) key.mv_size,  (char *) key.mv_data,
+				data.mv_data, (int) data.mv_size, (char *) data.mv_data);
+		}
+		mdb_cursor_close(cur2);
+		E(mdb_txn_commit(txn));
+
+		printf("Restarting cursor outside txn\n");
+		E(mdb_txn_begin(env, NULL, 0, &txn));
+		E(mdb_cursor_open(txn, dbi, &cursor));
+		for (op=MDB_FIRST, i=0; i<=32; op=MDB_NEXT, i++) {
+			if (RES(MDB_NOTFOUND, mdb_cursor_get(cursor, &key, &data, op)))
+				break;
+			printf("key: %p %.*s, data: %p %.*s\n",
+				key.mv_data,  (int) key.mv_size,  (char *) key.mv_data,
+				data.mv_data, (int) data.mv_size, (char *) data.mv_data);
+		}
+		mdb_cursor_close(cursor);
+		mdb_txn_abort(txn);
+
+		mdb_dbi_close(env, dbi);
+		mdb_env_close(env);
+
+	return 0;
+}

From 34c3cfe8537e88a91cb8b580bbab52f716ec4559 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Fri, 30 Jun 2017 15:42:38 +0100
Subject: [PATCH 312/504] Plug some information leaks

Zero out decrypted pages before freeing them.
Do proper init on reused loose pages.
---
 libraries/liblmdb/mdb.c | 48 ++++++++++++++++++++++++++++++++---------
 1 file changed, 38 insertions(+), 10 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 6c7d8db456..c25f2d4f16 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1572,6 +1572,7 @@ static void mdb_txn_end(MDB_txn *txn, unsigned mode);
 #if MDB_RPAGE_CACHE
 static int  mdb_page_get(MDB_cursor *mc, pgno_t pgno, int numpgs, MDB_page **mp, int *lvl);
 #define MDB_PAGE_GET(mc, pg, numpgs, mp, lvl)	mdb_page_get(mc, pg, numpgs, mp, lvl)
+static void mdb_rpage_dispose(MDB_env *env, MDB_ID3 *id3);
 #else
 static int  mdb_page_get(MDB_cursor *mc, pgno_t pgno, MDB_page **mp, int *lvl);
 #define MDB_PAGE_GET(mc, pg, numpgs, mp, lvl)	mdb_page_get(mc, pg, mp, lvl)
@@ -1998,19 +1999,20 @@ mdb_page_malloc(MDB_txn *txn, unsigned num, int init)
 	 * many pages they will be filling in at least up to the last page.
 	 */
 	if (num == 1) {
+		psize -= off = PAGEHDRSZ;
 		if (ret) {
 			VGMEMP_ALLOC(env, ret, sz);
 			VGMEMP_DEFINED(ret, sizeof(ret->mp_next));
 			env->me_dpages = ret->mp_next;
-			return ret;
+			goto init;
 		}
-		psize -= off = PAGEHDRSZ;
 	} else {
 		sz *= num;
 		off = sz - psize;
 	}
 	if ((ret = malloc(sz)) != NULL) {
 		VGMEMP_ALLOC(env, ret, sz);
+init:
 		if (init && !(env->me_flags & MDB_NOMEMINIT)) {
 			memset((char *)ret + off, 0, psize);
 			ret->mp_pad = 0;
@@ -3393,7 +3395,7 @@ mdb_txn_end(MDB_txn *txn, unsigned mode)
 				/* tmp overflow pages that we didn't share in env */
 				munmap(tl[i].mptr, tl[i].mcnt * env->me_psize);
 				if (tl[i].menc) {
-					free(tl[i].menc);
+					mdb_rpage_dispose(env, &tl[i]);
 					tl[i].menc = NULL;
 				}
 			} else {
@@ -3406,7 +3408,7 @@ mdb_txn_end(MDB_txn *txn, unsigned mode)
 					/* another tmp overflow page */
 					munmap(tl[i].mptr, tl[i].mcnt * env->me_psize);
 					if (tl[i].menc) {
-						free(tl[i].menc);
+						mdb_rpage_dispose(env, &tl[i]);
 						tl[i].menc = NULL;
 					}
 				}
@@ -6130,7 +6132,7 @@ mdb_rpage_get(MDB_txn *txn, pgno_t pg0, int numpgs, MDB_page **ret)
 				tl[x].mptr = id3.mptr;
 				tl[x].mcnt = id3.mcnt;
 				if (tl[x].menc)
-					free(tl[x].menc);
+					mdb_rpage_dispose(env, &tl[x]);
 				tl[x].menc = id3.menc;
 				tl[x].muse = id3.muse;
 				/* if no active ref, see if we can replace in env */
@@ -6144,7 +6146,7 @@ mdb_rpage_get(MDB_txn *txn, pgno_t pg0, int numpgs, MDB_page **ret)
 						el[i].mptr = tl[x].mptr;
 						el[i].mcnt = tl[x].mcnt;
 						if (el[i].menc)
-							free(el[i].menc);
+							mdb_rpage_dispose(env, &el[i]);
 						el[i].menc = tl[x].menc;
 						el[i].muse = tl[x].muse;
 					} else {
@@ -6181,7 +6183,7 @@ retry:
 				if (tl[i].mid & (MDB_RPAGE_CHUNK-1)) {
 					munmap(tl[i].mptr, tl[i].mcnt * env->me_psize);
 					if (tl[i].menc)
-						free(tl[i].menc);
+						mdb_rpage_dispose(env, &tl[i]);
 					continue;
 				}
 				x = mdb_mid3l_search(el, tl[i].mid);
@@ -6250,7 +6252,7 @@ retry:
 					el[x].mptr = id3.mptr;
 					el[x].mcnt = id3.mcnt;
 					if (el[x].menc)
-						free(el[x].menc);
+						mdb_rpage_dispose(env, &el[x]);
 					el[x].menc = id3.menc;
 					el[x].muse = id3.muse;
 				} else {
@@ -6279,7 +6281,7 @@ retry:
 					if (!y) y = i;
 					munmap(el[i].mptr, env->me_psize * el[i].mcnt);
 					if (el[i].menc)
-						free(el[i].menc);
+						mdb_rpage_dispose(env, &el[i]);
 				}
 			}
 			if (!y) {
@@ -6345,7 +6347,15 @@ static void mdb_rpage_decrypt(MDB_env *env, MDB_ID3 *id3, int rem, int numpgs)
 {
 	if (!(id3->muse & (1 << rem))) {
 		MDB_val in, out;
-		id3->muse |= (1 << rem);
+		int bit;
+
+		/* If this is an overflow page, set all use bits to the end */
+		if (rem + numpgs > MDB_RPAGE_CHUNK)
+			bit = 0xffff;
+		else
+			bit = 1;
+
+		id3->muse |= (bit << rem);
 		in.mv_size = numpgs * env->me_psize;
 		in.mv_data = (char *)id3->mptr + rem * env->me_psize;
 		out.mv_size = in.mv_size;
@@ -6353,6 +6363,24 @@ static void mdb_rpage_decrypt(MDB_env *env, MDB_ID3 *id3, int rem, int numpgs)
 		env->me_encfunc(&in, &out, env->me_enckey, 0);
 	}
 }
+
+/** zero out decrypted pages before freeing them */
+static void mdb_rpage_dispose(MDB_env *env, MDB_ID3 *id3)
+{
+	char *base = id3->menc;
+	int i, j;
+	for (i=0, j=1; i<15; i++) {
+		if (id3->muse & j)
+			memset(base, 0, env->me_psize);
+		j <<= 1;
+		base += env->me_psize;
+	}
+	if (id3->muse & j) {
+		i = id3->mcnt - (MDB_RPAGE_CHUNK - 1);
+		memset(base, 0, i * env->me_psize);
+	}
+	free(id3->menc);
+}
 #endif
 
 /** Find the address of the page corresponding to a given page number.

From fafbd42cd28a07c4dc88f363719e7f0d56c0f304 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Fri, 30 Jun 2017 16:30:29 +0100
Subject: [PATCH 313/504] Save/restore enc IV in page 0

---
 libraries/liblmdb/mdb.c | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index c25f2d4f16..2e91fece3d 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -4213,6 +4213,15 @@ mdb_env_init_meta(MDB_env *env, MDB_meta *meta)
 	q->mp_flags = P_META;
 	*(MDB_meta *)METADATA(q) = *meta;
 
+#if MDB_RPAGE_CACHE
+	if ((env->me_flags & MDB_ENCRYPT) && env->me_enckey[1].mv_size) {
+		/* save the IV in tail of page 0 */
+		char *ptr = (char *)q;
+		unsigned short *u = (unsigned short *)(ptr-2);
+		*u = env->me_enckey[1].mv_size;
+		memcpy(ptr - 2 - env->me_enckey[1].mv_size, env->me_enckey[1].mv_data, env->me_enckey[1].mv_size);
+	}
+#endif
 	DO_PWRITE(rc, env->me_fd, p, psize * NUM_METAS, len, 0);
 	if (!rc)
 		rc = ErrCode();
@@ -4914,6 +4923,21 @@ mdb_env_open2(MDB_env *env)
 	if ((env->me_flags ^ env->me_metas[0]->mm_flags) & MDB_ENCRYPT)
 		return MDB_INCOMPATIBLE;
 
+#if MDB_RPAGE_CACHE
+	if (!newenv && env->me_flags & MDB_ENCRYPT) {
+		/* for encrypted env, read IV from tail of page 0 */
+		char *ptr = env->me_map + env->me_psize, *ekey;
+		unsigned short *u = (unsigned short *)(ptr - 2);
+		env->me_enckey[1].mv_size = *u;
+		ekey = realloc(env->me_enckey[0].mv_data, env->me_enckey[0].mv_size + env->me_enckey[1].mv_size);
+		if (!ekey)
+			return ENOMEM;
+		env->me_enckey[0].mv_data = ekey;
+		env->me_enckey[1].mv_data = ekey + env->me_enckey[0].mv_size;
+		memcpy(env->me_enckey[1].mv_data, ptr - 2 - env->me_enckey[1].mv_size, env->me_enckey[1].mv_size);
+	}
+#endif
+
 	env->me_maxfree_1pg = (env->me_psize - PAGEHDRSZ) / sizeof(pgno_t) - 1;
 	env->me_nodemax = (((env->me_psize - PAGEHDRSZ) / MDB_MINKEYS) & -2)
 		- sizeof(indx_t);

From de161fe9d963fa3b2af4e60f861ef6e0d2d607c3 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 1 Jul 2017 13:15:37 +0200
Subject: [PATCH 314/504] Get flags from $LMDB_FLAGS when MDB_TEST

Now we don't need to tweak the code of callers to test encryption.
---
 libraries/liblmdb/mdb.c | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 2e91fece3d..d6260d329c 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5463,6 +5463,34 @@ fail:
 	return rc;
 }
 
+#ifdef MDB_TEST
+/** Add #mdb_env_open() flags from environment variable $LMDB_FLAGS.
+ */
+static int ESECT
+mdb_env_getflags(MDB_env *env)
+{
+	static const char names[] = "acfhilmnrstw";
+	static const unsigned f[] = {
+		MDB_MAPASYNC, MDB_REMAP_CHUNKS, MDB_FIXEDMAP, MDB_NORDAHEAD,
+		MDB_NOMEMINIT, MDB_NOLOCK, MDB_NOMETASYNC, MDB_NOSUBDIR,
+		MDB_RDONLY, MDB_NOSYNC, MDB_NOTLS, MDB_WRITEMAP,
+	};
+	unsigned flags = 0;
+	const char *s, *opts = getenv("LMDB_FLAGS");
+	if (opts) {
+		for (; *opts; opts++) {
+			if ((s = strchr(names, *opts)) == NULL)
+				return EINVAL;
+			flags |= f[s - names];
+		}
+		env->me_flags |= flags;
+	}
+	return MDB_SUCCESS;
+}
+#else
+#define mdb_env_getflags(env) MDB_SUCCESS
+#endif	/* MDB_TEST */
+
 	/** Only a subset of the @ref mdb_env flags can be changed
 	 *	at runtime. Changing other flags requires closing the
 	 *	environment and re-opening it with the new flags.
@@ -5484,7 +5512,10 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 	if (env->me_fd!=INVALID_HANDLE_VALUE || (flags & ~(CHANGEABLE|CHANGELESS)))
 		return EINVAL;
 
+	if ((rc = mdb_env_getflags(env)) != MDB_SUCCESS)
+		return rc;
 	flags |= env->me_flags;
+
 	if (MDB_REMAPPING(0))			/* if we always remap chunks */
 		flags |= MDB_REMAP_CHUNKS;
 	if (MDB_REMAPPING(flags)) {

From f16ce88dc8bb6426ad824518eded2f691d60cc94 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 1 Jul 2017 08:04:59 +0200
Subject: [PATCH 315/504] Trivial encryption when $LMDB_FLAGS = "e"

---
 libraries/liblmdb/mdb.c | 35 ++++++++++++++++++++++++++++++++++-
 1 file changed, 34 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index d6260d329c..90160fed81 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5464,16 +5464,33 @@ fail:
 }
 
 #ifdef MDB_TEST
+#if MDB_RPAGE_CACHE
+/** Trivial encryption for testing */
+static void ESECT
+mdb_enctest(const MDB_val *src, MDB_val *dst, const MDB_val *key, int encdec)
+{
+	mdb_size_t *sptr = src->mv_data, *dptr = dst->mv_data;
+	mdb_size_t x=*(mdb_size_t*)key[0].mv_data, v=*(mdb_size_t*)key[1].mv_data;
+	int i = dst->mv_size / sizeof(mdb_size_t);
+
+	while (--i >= 0)
+		x += v += i + sptr[i] + (dptr[i] = sptr[i] ^ x);
+}
+#endif	/* MDB_RPAGE_CACHE */
+
 /** Add #mdb_env_open() flags from environment variable $LMDB_FLAGS.
+ *
+ *	Supports the normal flags plus 'e' = trivial encryption for testing.
  */
 static int ESECT
 mdb_env_getflags(MDB_env *env)
 {
-	static const char names[] = "acfhilmnrstw";
+	static const char names[] = "acfhilmnrstwe";
 	static const unsigned f[] = {
 		MDB_MAPASYNC, MDB_REMAP_CHUNKS, MDB_FIXEDMAP, MDB_NORDAHEAD,
 		MDB_NOMEMINIT, MDB_NOLOCK, MDB_NOMETASYNC, MDB_NOSUBDIR,
 		MDB_RDONLY, MDB_NOSYNC, MDB_NOTLS, MDB_WRITEMAP,
+		MDB_ENCRYPT,
 	};
 	unsigned flags = 0;
 	const char *s, *opts = getenv("LMDB_FLAGS");
@@ -5483,6 +5500,22 @@ mdb_env_getflags(MDB_env *env)
 				return EINVAL;
 			flags |= f[s - names];
 		}
+		if (flags & MDB_ENCRYPT) {
+#if MDB_RPAGE_CACHE
+			if (!env->me_encfunc) {
+				static mdb_size_t k = MDB_SIZE_MAX/9*37;
+				mdb_size_t iv = ((mdb_size_t)env ^ env->me_pid);
+				MDB_val keys[2] = { {sizeof(k), &k}, {sizeof(iv), NULL} };
+				int rc;
+				keys[1].mv_data = &iv;
+				rc = mdb_env_set_encrypt(env, mdb_enctest, keys);
+				if (rc)
+					return rc;
+			}
+#else
+			return EINVAL;
+#endif
+		}
 		env->me_flags |= flags;
 	}
 	return MDB_SUCCESS;

From c7ef535aa92828bae105538f0aabd3e261ad945a Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sun, 2 Jul 2017 17:13:48 +0100
Subject: [PATCH 316/504] Data format change: add txnid to page header

---
 libraries/liblmdb/mdb.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 90160fed81..2c3454ca62 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -619,7 +619,7 @@ static txnid_t mdb_debug_start;
 #define MDB_MAGIC	 0xBEEFC0DE
 
 	/**	The version number for a database's datafile format. */
-#define MDB_DATA_VERSION	 ((MDB_DEVEL) ? 999 : 1)
+#define MDB_DATA_VERSION	 ((MDB_DEVEL) ? 999 : 2)
 	/**	The version number for a database's lockfile format. */
 #define MDB_LOCK_VERSION	 ((MDB_DEVEL) ? 999 : 2)
 	/** Number of bits representing #MDB_LOCK_VERSION in #MDB_LOCK_FORMAT.
@@ -961,6 +961,7 @@ typedef struct MDB_page {
 		pgno_t		p_pgno;	/**< page number */
 		struct MDB_page *p_next; /**< for in-memory list of freed pages */
 	} mp_p;
+	txnid_t		mp_txnid;		/**< txnid that committed this page, unused in meta pages */
 	uint16_t	mp_pad;			/**< key size if this is a LEAF2 page */
 /**	@defgroup mdb_page	Page Flags
  *	@ingroup internal
@@ -2625,6 +2626,7 @@ search_done:
 		txn->mt_next_pgno = pgno + num;
 	}
 	np->mp_pgno = pgno;
+	np->mp_txnid = txn->mt_txnid;
 	mdb_page_dirty(txn, np);
 	*mp = np;
 
@@ -2795,6 +2797,7 @@ mdb_page_touch(MDB_cursor *mc)
 	np->mp_flags |= P_DIRTY;
 
 done:
+	np->mp_txnid = txn->mt_txnid;
 	/* Adjust cursors pointing to mp */
 	mc->mc_pg[mc->mc_top] = np;
 	m2 = txn->mt_cursors[mc->mc_dbi];
@@ -7840,6 +7843,7 @@ more:
 			unsigned	i, offset = 0;
 			mp = fp = xdata.mv_data = env->me_pbuf;
 			mp->mp_pgno = mc->mc_pg[mc->mc_top]->mp_pgno;
+			mp->mp_txnid = mc->mc_txn->mt_txnid;
 
 			/* Was a single item before, must convert now */
 			if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) {
@@ -10362,6 +10366,7 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags)
 						mo = (MDB_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]);
 						memcpy(mo, omp, my->mc_env->me_psize);
 						mo->mp_pgno = my->mc_next_pgno;
+						mo->mp_txnid = 1;
 						my->mc_next_pgno += omp->mp_pages;
 						my->mc_wlen[toggle] += my->mc_env->me_psize;
 						if (dpages > 1) {
@@ -10426,6 +10431,7 @@ again:
 		mo = (MDB_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]);
 		mdb_page_copy(mo, mp, my->mc_env->me_psize);
 		mo->mp_pgno = my->mc_next_pgno++;
+		mo->mp_txnid = 1;
 		my->mc_wlen[toggle] += my->mc_env->me_psize;
 		if (mc.mc_top) {
 			/* Update parent if there is one */

From f0f985fa93259b0533455acef8a68d3bfef98e01 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 17 Jul 2017 13:02:00 +0100
Subject: [PATCH 317/504] checkpoint - moving overflow page headers

Moving headers outside of overflow page.
---
 libraries/liblmdb/mdb.c  | 195 +++++++++++++++++++++++----------------
 libraries/liblmdb/midl.c |  31 +++++++
 libraries/liblmdb/midl.h |   6 ++
 3 files changed, 153 insertions(+), 79 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 2c3454ca62..1e53764aa8 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -942,8 +942,9 @@ enum {
  * sorted #mp_ptrs[] entries referring to them. Exception: #P_LEAF2 pages
  * omit mp_ptrs and pack sorted #MDB_DUPFIXED values after the page header.
  *
- * #P_OVERFLOW records occupy one or more contiguous pages where only the
- * first has a page header. They hold the real data of #F_BIGDATA nodes.
+ * #P_OVERFLOW records occupy one or more contiguous pages that contain
+ * pure data with no page header. They hold the real data of #F_BIGDATA nodes,
+ * and the node stores what would have gone in a page header.
  *
  * #P_SUBP sub-pages are small leaf "pages" with duplicate data.
  * A node with flag #F_DUPDATA but not #F_SUBDATA contains a sub-page.
@@ -954,15 +955,15 @@ enum {
  * Each non-metapage up to #MDB_meta.%mm_last_pg is reachable exactly once
  * in the snapshot: Either used by a database or listed in a freeDB record.
  */
-typedef struct MDB_page {
+typedef struct MDB_page_header {
 #define	mp_pgno	mp_p.p_pgno
 #define	mp_next	mp_p.p_next
 	union {
 		pgno_t		p_pgno;	/**< page number */
 		struct MDB_page *p_next; /**< for in-memory list of freed pages */
-	} mp_p;
-	txnid_t		mp_txnid;		/**< txnid that committed this page, unused in meta pages */
-	uint16_t	mp_pad;			/**< key size if this is a LEAF2 page */
+	} mh_p;
+	txnid_t		mh_txnid;		/**< txnid that committed this page, unused in meta pages */
+	uint16_t	mh_pad;			/**< key size if this is a LEAF2 page */
 /**	@defgroup mdb_page	Page Flags
  *	@ingroup internal
  *	Flags for the page headers.
@@ -975,10 +976,11 @@ typedef struct MDB_page {
 #define	P_DIRTY		 0x10		/**< dirty page, also set for #P_SUBP pages */
 #define	P_LEAF2		 0x20		/**< for #MDB_DUPFIXED records */
 #define	P_SUBP		 0x40		/**< for #MDB_DUPSORT sub-pages */
+#define	P_DIRTY_OVF	 0x2000		/**< page has dirty overflow nodes */
 #define	P_LOOSE		 0x4000		/**< page was dirtied then freed, can be reused */
 #define	P_KEEP		 0x8000		/**< leave this page alone during spill */
 /** @} */
-	uint16_t	mp_flags;		/**< @ref mdb_page */
+	uint16_t	mh_flags;		/**< @ref mdb_page */
 #define mp_lower	mp_pb.pb.pb_lower
 #define mp_upper	mp_pb.pb.pb_upper
 #define mp_pages	mp_pb.pb_pages
@@ -988,12 +990,21 @@ typedef struct MDB_page {
 			indx_t		pb_upper;		/**< upper bound of free space */
 		} pb;
 		uint32_t	pb_pages;	/**< number of overflow pages */
-	} mp_pb;
+	} mh_pb;
+} MDB_page_header;
+
+typedef struct MDB_page {
+	MDB_page_header	mp_hdr;
+#define mp_p	mp_hdr.mh_p
+#define mp_txnid	mp_hdr.mh_txnid
+#define mp_pad		mp_hdr.mh_pad
+#define mp_flags	mp_hdr.mh_flags
+#define mp_pb		mp_hdr.mh_pb
 	indx_t		mp_ptrs[1];		/**< dynamic size */
 } MDB_page;
 
 	/** Size of the page header, excluding dynamic data at the end */
-#define PAGEHDRSZ	 ((unsigned) offsetof(MDB_page, mp_ptrs))
+#define PAGEHDRSZ	 sizeof(MDB_page_header)
 
 	/** Address of first usable data byte in a page, after the header */
 #define METADATA(p)	 ((void *)((char *)(p) + PAGEHDRSZ))
@@ -1026,6 +1037,19 @@ typedef struct MDB_page {
 	/** Test if a page is a sub page */
 #define IS_SUBP(p)	 F_ISSET((p)->mp_flags, P_SUBP)
 
+	/** Header for overflow pages, stored in an F_BIGDATA node */
+typedef struct MDB_ovpage {
+	pgno_t	op_pgno;
+	txnid_t	op_txnid;
+	mdb_size_t	op_pages;
+} MDB_ovpage;
+
+	/** Header for a dirty overflow page in memory */
+typedef struct MDB_dovpage {
+	MDB_page_header mp_hdr;
+	void	*mp_ptr;
+} MDB_dovpage;
+
 	/** The number of overflow pages needed to store the given size. */
 #define OVPAGES(size, psize)	((PAGEHDRSZ-1 + (size)) / (psize) + 1)
 
@@ -1044,7 +1068,7 @@ typedef struct MDB_page {
 	 * order in case some accesses can be optimized to 32-bit word access.
 	 *
 	 * Leaf node flags describe node contents.  #F_BIGDATA says the node's
-	 * data part is the page number of an overflow page with actual data.
+	 * data part is an MDB_ovpage struct pointing to a page with actual data.
 	 * #F_DUPDATA and #F_SUBDATA can be combined giving duplicate data in
 	 * a sub-page/sub-database, and named databases (just #F_SUBDATA).
 	 */
@@ -1276,6 +1300,8 @@ struct MDB_txn {
 		/** For read txns: This thread/txn's reader table slot, or NULL. */
 		MDB_reader	*reader;
 	} mt_u;
+	/** The sorted list of dirty overflow pages. */
+	MDB_ID2L	mt_dirty_ovs;
 	/** Array of records for each DB known in the environment. */
 	MDB_dbx		*mt_dbxs;
 	/** Array of MDB_db records for each known DB */
@@ -1553,7 +1579,7 @@ typedef struct MDB_ntxn {
 #define TXN_DBI_CHANGED(txn, dbi) \
 	((txn)->mt_dbiseqs[dbi] != (txn)->mt_env->me_dbiseqs[dbi])
 
-static int  mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp);
+static int  mdb_page_alloc(MDB_cursor *mc, int num, int ov, MDB_page **mp);
 static int  mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp);
 static int  mdb_page_touch(MDB_cursor *mc);
 
@@ -1871,7 +1897,7 @@ mdb_page_list(MDB_page *mp)
 			total += nsize;
 		} else {
 			if (F_ISSET(node->mn_flags, F_BIGDATA))
-				nsize += sizeof(pgno_t);
+				nsize += sizeof(MDB_ovpage);
 			else
 				nsize += NODEDSZ(node);
 			total += nsize;
@@ -2409,7 +2435,7 @@ mdb_find_oldest(MDB_txn *txn)
 
 /** Add a page to the txn's dirty list */
 static void
-mdb_page_dirty(MDB_txn *txn, MDB_page *mp)
+mdb_page_dirty(MDB_txn *txn, MDB_page *mp, int ov)
 {
 	MDB_ID2 mid;
 	int rc, (*insert)(MDB_ID2L, MDB_ID2 *);
@@ -2421,9 +2447,13 @@ mdb_page_dirty(MDB_txn *txn, MDB_page *mp)
 	}
 	mid.mid = mp->mp_pgno;
 	mid.mptr = mp;
-	rc = insert(txn->mt_u.dirty_list, &mid);
+	if (ov) {
+		rc = mdb_mid2l_insert(txn->mt_dirty_ovs, &mid);
+	} else {
+		rc = insert(txn->mt_u.dirty_list, &mid);
+		txn->mt_dirty_room--;
+	}
 	mdb_tassert(txn, rc == 0);
-	txn->mt_dirty_room--;
 }
 
 /** Allocate page numbers and memory for writing.  Maintain me_pglast,
@@ -2444,7 +2474,7 @@ mdb_page_dirty(MDB_txn *txn, MDB_page *mp)
  * @return 0 on success, non-zero on failure.
  */
 static int
-mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
+mdb_page_alloc(MDB_cursor *mc, int num, int ov, MDB_page **mp)
 {
 #ifdef MDB_PARANOID	/* Seems like we can ignore this now */
 	/* Get at most <Max_retries> more freeDB records once me_pghead
@@ -2467,6 +2497,17 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
 	MDB_cursor_op op;
 	MDB_cursor m2;
 	int found_old = 0;
+	MDB_dovpage *dph = NULL;
+
+	if (ov) {
+		if (!txn->mt_dirty_ovs) {
+			txn->mt_dirty_ovs = mdb_mid2l_alloc(16);
+			if (!txn->mt_dirty_ovs)
+				return ENOMEM;
+		} else if (mdb_mid2l_need(&txn->mt_dirty_ovs, txn->mt_dirty_ovs[0].mid + 1))
+			return ENOMEM;
+		dph = malloc(sizeof(MDB_dovpage));
+	}
 
 	/* If there are any loose pages, just use them */
 	if (num == 1 && txn->mt_loose_pgs) {
@@ -2474,6 +2515,11 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
 		txn->mt_loose_pgs = NEXT_LOOSE_PAGE(np);
 		txn->mt_loose_count--;
 		DPRINTF(("db %d use loose page %"Yu, DDBI(mc), np->mp_pgno));
+		if (ov) {
+			dph->mp_hdr = np->mp_hdr;
+			dph->mp_ptr = np;
+			np = (MDB_page *)dph;
+		}
 		*mp = np;
 		return MDB_SUCCESS;
 	}
@@ -2627,12 +2673,19 @@ search_done:
 	}
 	np->mp_pgno = pgno;
 	np->mp_txnid = txn->mt_txnid;
-	mdb_page_dirty(txn, np);
+	if (ov) {
+		dph->mp_hdr = np->mp_hdr;
+		dph->mp_ptr = np;
+		np = (MDB_page *)dph;
+	}
+	mdb_page_dirty(txn, np, ov);
 	*mp = np;
 
 	return MDB_SUCCESS;
 
 fail:
+	if (dph)
+		free(dph);
 	txn->mt_flags |= MDB_TXN_ERROR;
 	return rc;
 }
@@ -2670,7 +2723,7 @@ mdb_page_copy(MDB_page *dst, MDB_page *src, unsigned int psize)
  * mp wasn't spilled.
  */
 static int
-mdb_page_unspill(MDB_txn *txn, MDB_page *mp, MDB_page **ret)
+mdb_page_unspill(MDB_txn *txn, MDB_page *mp, int num, int ov, MDB_page **ret)
 {
 	MDB_env *env = txn->mt_env;
 	const MDB_txn *tx2;
@@ -2683,20 +2736,15 @@ mdb_page_unspill(MDB_txn *txn, MDB_page *mp, MDB_page **ret)
 		x = mdb_midl_search(tx2->mt_spill_pgs, pn);
 		if (x <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[x] == pn) {
 			MDB_page *np;
-			int num;
 			if (txn->mt_dirty_room == 0)
 				return MDB_TXN_FULL;
-			if (IS_OVERFLOW(mp))
-				num = mp->mp_pages;
-			else
-				num = 1;
 			if (env->me_flags & MDB_WRITEMAP) {
 				np = mp;
 			} else {
 				np = mdb_page_malloc(txn, num, 1);
 				if (!np)
 					return ENOMEM;
-				if (num > 1)
+				if (ov)
 					memcpy(np, mp, num * env->me_psize);
 				else
 					mdb_page_copy(np, mp, env->me_psize);
@@ -2714,8 +2762,9 @@ mdb_page_unspill(MDB_txn *txn, MDB_page *mp, MDB_page **ret)
 				 * page remains spilled until child commits
 				 */
 
-			mdb_page_dirty(txn, np);
-			np->mp_flags |= P_DIRTY;
+			mdb_page_dirty(txn, np, ov);
+			if (!ov)
+				np->mp_flags |= P_DIRTY;
 			*ret = np;
 			break;
 		}
@@ -2740,14 +2789,14 @@ mdb_page_touch(MDB_cursor *mc)
 	if (!F_ISSET(mp->mp_flags, P_DIRTY)) {
 		if (txn->mt_flags & MDB_TXN_SPILLS) {
 			np = NULL;
-			rc = mdb_page_unspill(txn, mp, &np);
+			rc = mdb_page_unspill(txn, mp, 1, 0, &np);
 			if (rc)
 				goto fail;
 			if (np)
 				goto done;
 		}
 		if ((rc = mdb_midl_need(&txn->mt_free_pgs, 1)) ||
-			(rc = mdb_page_alloc(mc, 1, &np)))
+			(rc = mdb_page_alloc(mc, 1, 0, &np)))
 			goto fail;
 		pgno = np->mp_pgno;
 		DPRINTF(("touched db %d page %"Yu" -> %"Yu, DDBI(mc),
@@ -3099,6 +3148,7 @@ mdb_txn_renew0(MDB_txn *txn)
 		txn->mt_free_pgs = env->me_free_pgs;
 		txn->mt_free_pgs[0] = 0;
 		txn->mt_spill_pgs = NULL;
+		txn->mt_dirty_ovs = NULL;
 		env->me_txn = txn;
 		memcpy(txn->mt_dbiseqs, env->me_dbiseqs, env->me_maxdbs * sizeof(unsigned int));
 	}
@@ -3223,6 +3273,7 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 		txn->mt_dirty_room = parent->mt_dirty_room;
 		txn->mt_u.dirty_list[0].mid = 0;
 		txn->mt_spill_pgs = NULL;
+		txn->mt_dirty_ovs = NULL;
 		txn->mt_next_pgno = parent->mt_next_pgno;
 		parent->mt_flags |= MDB_TXN_HAS_CHILD;
 		parent->mt_child = txn;
@@ -3383,6 +3434,7 @@ mdb_txn_end(MDB_txn *txn, unsigned mode)
 			env->me_pgstate = ((MDB_ntxn *)txn)->mnt_pgstate;
 			mdb_midl_free(txn->mt_free_pgs);
 			mdb_midl_free(txn->mt_spill_pgs);
+			mdb_mid2l_free(txn->mt_dirty_ovs);
 			free(txn->mt_u.dirty_list);
 		}
 
@@ -6843,7 +6895,7 @@ static int
 mdb_node_read(MDB_cursor *mc, MDB_node *leaf, MDB_val *data)
 {
 	MDB_page	*omp;		/* overflow page */
-	pgno_t		 pgno;
+	MDB_ovpage	ovp;
 	int rc;
 
 	if (MC_OVPG(mc)) {
@@ -6858,13 +6910,10 @@ mdb_node_read(MDB_cursor *mc, MDB_node *leaf, MDB_val *data)
 
 	/* Read overflow data.
 	 */
-	memcpy(&pgno, NODEDATA(leaf), sizeof(pgno));
+	memcpy(&ovp, NODEDATA(leaf), sizeof(ovp));
 	{
-#if MDB_RPAGE_CACHE
-	int dpages = OVPAGES(data->mv_size, mc->mc_txn->mt_env->me_psize);
-#endif
-	if ((rc = MDB_PAGE_GET(mc, pgno, dpages, &omp, NULL)) != 0) {
-		DPRINTF(("read overflow page %"Yu" failed", pgno));
+	if ((rc = MDB_PAGE_GET(mc, ovp.op_pgno, ovp.op_pages, &omp, NULL)) != 0) {
+		DPRINTF(("read overflow page %"Yu" failed", ovp.op_pgno));
 		return rc;
 	}
 	}
@@ -7932,7 +7981,7 @@ prep_subDB:
 					dummy.md_entries = NUMKEYS(fp);
 					xdata.mv_size = sizeof(MDB_db);
 					xdata.mv_data = &dummy;
-					if ((rc = mdb_page_alloc(mc, 1, &mp)))
+					if ((rc = mdb_page_alloc(mc, 1, 0, &mp)))
 						return rc;
 					offset = env->me_psize - olddata.mv_size;
 					flags |= F_DUPDATA|F_SUBDATA;
@@ -7968,26 +8017,27 @@ current:
 		/* overflow page overwrites need special handling */
 		if (F_ISSET(leaf->mn_flags, F_BIGDATA)) {
 			MDB_page *omp;
-			pgno_t pg;
+			MDB_ovpage ovp;
 			int level, ovpages, dpages = OVPAGES(data->mv_size, env->me_psize);
 
-			memcpy(&pg, olddata.mv_data, sizeof(pg));
-			if ((rc2 = MDB_PAGE_GET(mc, pg, dpages, &omp, &level)) != 0)
+			memcpy(&ovp, olddata.mv_data, sizeof(ovp));
+			if ((rc2 = MDB_PAGE_GET(mc, ovp.op_pgno, ovp.op_pages, &omp, &level)) != 0)
 				return rc2;
-			ovpages = omp->mp_pages;
+			ovpages = ovp.op_pages;
 
 			/* Is the ov page large enough? */
 			if (ovpages >= dpages) {
+			  /* Did we dirty it in this txn? */
 			  if (!(omp->mp_flags & P_DIRTY) &&
 				  (level || (env->me_flags & MDB_WRITEMAP)))
 			  {
-				rc = mdb_page_unspill(mc->mc_txn, omp, &omp);
+				rc = mdb_page_unspill(mc->mc_txn, omp, ovpages, 1, &omp);
 				if (rc)
 					return rc;
 				level = 0;		/* dirty in this txn or clean */
 			  }
 			  /* Is it dirty? */
-			  if (omp->mp_flags & P_DIRTY) {
+			  if (ovp.op_txnid == mc->mc_txn->mt_txnid) {
 				/* yes, overwrite it. Note in this case we don't
 				 * bother to try shrinking the page if the new data
 				 * is smaller than the overflow threshold.
@@ -7999,7 +8049,7 @@ current:
 					MDB_ID2 id2;
 					if (!np)
 						return ENOMEM;
-					id2.mid = pg;
+					id2.mid = ovp.op_pgno;
 					id2.mptr = np;
 					/* Note - this page is already counted in parent's dirty_room */
 					rc2 = mdb_mid2l_insert(mc->mc_txn->mt_u.dirty_list, &id2);
@@ -8013,12 +8063,10 @@ current:
 						 * Copy end of page, adjusting alignment so
 						 * compiler may copy words instead of bytes.
 						 */
-						off = (PAGEHDRSZ + data->mv_size) & -sizeof(size_t);
+						off = data->mv_size & -sizeof(size_t);
 						memcpy((size_t *)((char *)np + off),
 							(size_t *)((char *)omp + off), sz - off);
-						sz = PAGEHDRSZ;
 					}
-					memcpy(np, omp, sz); /* Copy beginning of page */
 					omp = np;
 				}
 				SETDSZ(leaf, data->mv_size);
@@ -8269,13 +8317,13 @@ mdb_cursor_del(MDB_cursor *mc, unsigned int flags)
 	/* add overflow pages to free list */
 	if (F_ISSET(leaf->mn_flags, F_BIGDATA)) {
 		MDB_page *omp;
-		pgno_t pg;
+		MDB_ovpage ovp;
 
-		memcpy(&pg, NODEDATA(leaf), sizeof(pg));
+		memcpy(&ovp, NODEDATA(leaf), sizeof(ovp));
 		/* note we don't care about page count here since
 		 * we're just adding pgno to the freelist anyway
 		 */
-		if ((rc = MDB_PAGE_GET(mc, pg, 1, &omp, NULL)) ||
+		if ((rc = MDB_PAGE_GET(mc, ovp.op_pgno, 1, &omp, NULL)) ||
 			(rc = mdb_ovpage_free(mc, omp)))
 			goto fail;
 	}
@@ -8303,7 +8351,7 @@ mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp)
 	MDB_page	*np;
 	int rc;
 
-	if ((rc = mdb_page_alloc(mc, num, &np)))
+	if ((rc = mdb_page_alloc(mc, num, flags & P_OVERFLOW, &np)))
 		return rc;
 	DPRINTF(("allocated new mpage %"Yu", page size %u",
 	    np->mp_pgno, mc->mc_txn->mt_env->me_psize));
@@ -8435,7 +8483,7 @@ mdb_node_add(MDB_cursor *mc, indx_t indx,
 		mdb_cassert(mc, key && data);
 		if (F_ISSET(flags, F_BIGDATA)) {
 			/* Data already on overflow page. */
-			node_size += sizeof(pgno_t);
+			node_size += sizeof(MDB_ovpage);
 		} else if (node_size + data->mv_size > mc->mc_txn->mt_env->me_nodemax) {
 			int ovpages = OVPAGES(data->mv_size, mc->mc_txn->mt_env->me_psize);
 			int rc;
@@ -8486,14 +8534,13 @@ update:
 		ndata = NODEDATA(node);
 		if (ofp == NULL) {
 			if (F_ISSET(flags, F_BIGDATA))
-				memcpy(ndata, data->mv_data, sizeof(pgno_t));
+				memcpy(ndata, data->mv_data, sizeof(MDB_ovpage));
 			else if (F_ISSET(flags, MDB_RESERVE))
 				data->mv_data = ndata;
 			else
 				memcpy(ndata, data->mv_data, data->mv_size);
 		} else {
-			memcpy(ndata, &ofp->mp_pgno, sizeof(pgno_t));
-			ndata = METADATA(ofp);
+			ndata = ((MDB_dovpage *)ofp)->mp_ptr;
 			if (F_ISSET(flags, MDB_RESERVE))
 				data->mv_data = ndata;
 			else
@@ -8546,7 +8593,7 @@ mdb_node_del(MDB_cursor *mc, int ksize)
 	sz = NODESIZE + node->mn_ksize;
 	if (IS_LEAF(mp)) {
 		if (F_ISSET(node->mn_flags, F_BIGDATA))
-			sz += sizeof(pgno_t);
+			sz += sizeof(MDB_ovpage);
 		else
 			sz += NODEDSZ(node);
 	}
@@ -9901,7 +9948,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 						psize += NODESIZE + NODEKSZ(node) + sizeof(indx_t);
 						if (IS_LEAF(mp)) {
 							if (F_ISSET(node->mn_flags, F_BIGDATA))
-								psize += sizeof(pgno_t);
+								psize += sizeof(MDB_ovpage);
 							else
 								psize += NODEDSZ(node);
 						}
@@ -10338,9 +10385,7 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags)
 					ni = NODEPTR(mp, i);
 					if (ni->mn_flags & F_BIGDATA) {
 						MDB_page *omp;
-						pgno_t pg;
-						size_t dsize;
-						int dpages;
+						MDB_ovpage ovp;
 
 						/* Need writable leaf */
 						if (mp != leaf) {
@@ -10349,12 +10394,9 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags)
 							mp = leaf;
 							ni = NODEPTR(mp, i);
 						}
-						dsize = NODEDSZ(ni);
-						dpages = OVPAGES(dsize, my->mc_env->me_psize);
 
-						memcpy(&pg, NODEDATA(ni), sizeof(pg));
-						memcpy(NODEDATA(ni), &my->mc_next_pgno, sizeof(pgno_t));
-						rc = MDB_PAGE_GET(&mc, pg, dpages, &omp, NULL);
+						memcpy(&ovp, NODEDATA(ni), sizeof(ovp));
+						rc = MDB_PAGE_GET(&mc, ovp.op_pgno, ovp.op_pages, &omp, NULL);
 						if (rc)
 							goto done;
 						if (my->mc_wlen[toggle] >= MDB_WBUF) {
@@ -10365,12 +10407,13 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags)
 						}
 						mo = (MDB_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]);
 						memcpy(mo, omp, my->mc_env->me_psize);
-						mo->mp_pgno = my->mc_next_pgno;
-						mo->mp_txnid = 1;
-						my->mc_next_pgno += omp->mp_pages;
+						ovp.op_pgno = my->mc_next_pgno;
+						ovp.op_txnid = 1;
+						memcpy(NODEDATA(ni), &ovp, sizeof(ovp));
+						my->mc_next_pgno += ovp.op_pages;
 						my->mc_wlen[toggle] += my->mc_env->me_psize;
-						if (dpages > 1) {
-							my->mc_olen[toggle] = my->mc_env->me_psize * (dpages - 1);
+						if (ovp.op_pages > 1) {
+							my->mc_olen[toggle] = my->mc_env->me_psize * (ovp.op_pages - 1);
 							my->mc_over[toggle] = (char *)omp + my->mc_env->me_psize;
 							rc = mdb_env_cthr_toggle(my, 1);
 							if (rc)
@@ -11089,19 +11132,13 @@ mdb_drop0(MDB_cursor *mc, int subs)
 				for (i=0; i<n; i++) {
 					ni = NODEPTR(mp, i);
 					if (ni->mn_flags & F_BIGDATA) {
-						MDB_page *omp;
-						pgno_t pg;
-						memcpy(&pg, NODEDATA(ni), sizeof(pg));
-						/* page count is irrelevant here */
-						rc = MDB_PAGE_GET(mc, pg, 1, &omp, NULL);
-						if (rc != 0)
-							goto done;
-						mdb_cassert(mc, IS_OVERFLOW(omp));
+						MDB_ovpage ovp;
+						memcpy(&ovp, NODEDATA(ni), sizeof(ovp));
 						rc = mdb_midl_append_range(&txn->mt_free_pgs,
-							pg, omp->mp_pages);
+							ovp.op_pgno, ovp.op_pages);
 						if (rc)
 							goto done;
-						mc->mc_db->md_overflow_pages -= omp->mp_pages;
+						mc->mc_db->md_overflow_pages -= ovp.op_pages;
 						if (!mc->mc_db->md_overflow_pages && !subs)
 							break;
 					} else if (subs && (ni->mn_flags & F_SUBDATA)) {
diff --git a/libraries/liblmdb/midl.c b/libraries/liblmdb/midl.c
index 05b91f7ed4..c5e5828a86 100644
--- a/libraries/liblmdb/midl.c
+++ b/libraries/liblmdb/midl.c
@@ -355,6 +355,37 @@ int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id )
 	return 0;
 }
 
+MDB_ID2L mdb_mid2l_alloc(int num)
+{
+	MDB_ID2L ids = malloc((num+2) * sizeof(MDB_ID2));
+	if (ids) {
+		ids->mid = num;
+		ids++;
+		ids->mid = 0;
+	}
+	return ids;
+}
+
+void mdb_mid2l_free(MDB_ID2L ids)
+{
+	if (ids)
+		free(ids-1);
+}
+
+int mdb_mid2l_need( MDB_ID2L *idp, unsigned num )
+{
+	MDB_ID2L ids = *idp;
+	num += ids[0].mid;
+	if (num > ids[-1].mid) {
+		num = (num + num/4 + (256 + 2)) & -256;
+		if (!(ids = realloc(ids-1, num * sizeof(MDB_ID2))))
+			return ENOMEM;
+		ids[0].mid = num - 2;
+		*idp = ids+1;
+	}
+	return 0;
+}
+
 #if MDB_RPAGE_CACHE
 unsigned mdb_mid3l_search( MDB_ID3L ids, MDB_ID id )
 {
diff --git a/libraries/liblmdb/midl.h b/libraries/liblmdb/midl.h
index 1a603da332..c814d45583 100644
--- a/libraries/liblmdb/midl.h
+++ b/libraries/liblmdb/midl.h
@@ -178,6 +178,12 @@ int mdb_mid2l_insert( MDB_ID2L ids, MDB_ID2 *id );
 	 */
 int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id );
 
+MDB_ID2L mdb_mid2l_alloc(int num);
+
+void mdb_mid2l_free(MDB_ID2L ids);
+
+int mdb_mid2l_need( MDB_ID2L *idp, unsigned num );
+
 #if MDB_RPAGE_CACHE
 typedef struct MDB_ID3 {
 	MDB_ID mid;		/**< The ID */

From 757378fc1d0b598f69d672f8338dcfa218216899 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 25 Jul 2017 15:34:03 +0100
Subject: [PATCH 318/504] Scaled back on overflow page work

Still keeping page header at top of overflow page for now
---
 libraries/liblmdb/mdb.c | 75 +++++++++++++++++++++++++++--------------
 1 file changed, 49 insertions(+), 26 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 1e53764aa8..ad5dce6b76 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -942,9 +942,9 @@ enum {
  * sorted #mp_ptrs[] entries referring to them. Exception: #P_LEAF2 pages
  * omit mp_ptrs and pack sorted #MDB_DUPFIXED values after the page header.
  *
- * #P_OVERFLOW records occupy one or more contiguous pages that contain
- * pure data with no page header. They hold the real data of #F_BIGDATA nodes,
- * and the node stores what would have gone in a page header.
+ * #P_OVERFLOW records occupy one or more contiguous pages where only the
+ * first has a page header. They hold the real data of #F_BIGDATA nodes,
+ * and the node stores the pgno and number of pages used by the record.
  *
  * #P_SUBP sub-pages are small leaf "pages" with duplicate data.
  * A node with flag #F_DUPDATA but not #F_SUBDATA contains a sub-page.
@@ -1004,7 +1004,7 @@ typedef struct MDB_page {
 } MDB_page;
 
 	/** Size of the page header, excluding dynamic data at the end */
-#define PAGEHDRSZ	 sizeof(MDB_page_header)
+#define PAGEHDRSZ	 ((unsigned)sizeof(MDB_page_header))
 
 	/** Address of first usable data byte in a page, after the header */
 #define METADATA(p)	 ((void *)((char *)(p) + PAGEHDRSZ))
@@ -1300,8 +1300,10 @@ struct MDB_txn {
 		/** For read txns: This thread/txn's reader table slot, or NULL. */
 		MDB_reader	*reader;
 	} mt_u;
+#if OVERFLOW_NOTYET
 	/** The sorted list of dirty overflow pages. */
 	MDB_ID2L	mt_dirty_ovs;
+#endif
 	/** Array of records for each DB known in the environment. */
 	MDB_dbx		*mt_dbxs;
 	/** Array of MDB_db records for each known DB */
@@ -1579,7 +1581,7 @@ typedef struct MDB_ntxn {
 #define TXN_DBI_CHANGED(txn, dbi) \
 	((txn)->mt_dbiseqs[dbi] != (txn)->mt_env->me_dbiseqs[dbi])
 
-static int  mdb_page_alloc(MDB_cursor *mc, int num, int ov, MDB_page **mp);
+static int  mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp);
 static int  mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp);
 static int  mdb_page_touch(MDB_cursor *mc);
 
@@ -2435,7 +2437,7 @@ mdb_find_oldest(MDB_txn *txn)
 
 /** Add a page to the txn's dirty list */
 static void
-mdb_page_dirty(MDB_txn *txn, MDB_page *mp, int ov)
+mdb_page_dirty(MDB_txn *txn, MDB_page *mp)
 {
 	MDB_ID2 mid;
 	int rc, (*insert)(MDB_ID2L, MDB_ID2 *);
@@ -2447,13 +2449,9 @@ mdb_page_dirty(MDB_txn *txn, MDB_page *mp, int ov)
 	}
 	mid.mid = mp->mp_pgno;
 	mid.mptr = mp;
-	if (ov) {
-		rc = mdb_mid2l_insert(txn->mt_dirty_ovs, &mid);
-	} else {
-		rc = insert(txn->mt_u.dirty_list, &mid);
-		txn->mt_dirty_room--;
-	}
+	rc = insert(txn->mt_u.dirty_list, &mid);
 	mdb_tassert(txn, rc == 0);
+	txn->mt_dirty_room--;
 }
 
 /** Allocate page numbers and memory for writing.  Maintain me_pglast,
@@ -2474,7 +2472,7 @@ mdb_page_dirty(MDB_txn *txn, MDB_page *mp, int ov)
  * @return 0 on success, non-zero on failure.
  */
 static int
-mdb_page_alloc(MDB_cursor *mc, int num, int ov, MDB_page **mp)
+mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
 {
 #ifdef MDB_PARANOID	/* Seems like we can ignore this now */
 	/* Get at most <Max_retries> more freeDB records once me_pghead
@@ -2497,6 +2495,8 @@ mdb_page_alloc(MDB_cursor *mc, int num, int ov, MDB_page **mp)
 	MDB_cursor_op op;
 	MDB_cursor m2;
 	int found_old = 0;
+
+#if OVERFLOW_NOTYET
 	MDB_dovpage *dph = NULL;
 
 	if (ov) {
@@ -2508,6 +2508,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, int ov, MDB_page **mp)
 			return ENOMEM;
 		dph = malloc(sizeof(MDB_dovpage));
 	}
+#endif
 
 	/* If there are any loose pages, just use them */
 	if (num == 1 && txn->mt_loose_pgs) {
@@ -2515,11 +2516,13 @@ mdb_page_alloc(MDB_cursor *mc, int num, int ov, MDB_page **mp)
 		txn->mt_loose_pgs = NEXT_LOOSE_PAGE(np);
 		txn->mt_loose_count--;
 		DPRINTF(("db %d use loose page %"Yu, DDBI(mc), np->mp_pgno));
+#if OVERFLOW_NOTYET
 		if (ov) {
 			dph->mp_hdr = np->mp_hdr;
 			dph->mp_ptr = np;
 			np = (MDB_page *)dph;
 		}
+#endif
 		*mp = np;
 		return MDB_SUCCESS;
 	}
@@ -2673,19 +2676,25 @@ search_done:
 	}
 	np->mp_pgno = pgno;
 	np->mp_txnid = txn->mt_txnid;
+#if OVERFLOW_NOTYET
 	if (ov) {
 		dph->mp_hdr = np->mp_hdr;
 		dph->mp_ptr = np;
 		np = (MDB_page *)dph;
 	}
 	mdb_page_dirty(txn, np, ov);
+#else
+	mdb_page_dirty(txn, np);
+#endif
 	*mp = np;
 
 	return MDB_SUCCESS;
 
 fail:
+#if OVERFLOW_NOTYET
 	if (dph)
 		free(dph);
+#endif
 	txn->mt_flags |= MDB_TXN_ERROR;
 	return rc;
 }
@@ -2723,7 +2732,7 @@ mdb_page_copy(MDB_page *dst, MDB_page *src, unsigned int psize)
  * mp wasn't spilled.
  */
 static int
-mdb_page_unspill(MDB_txn *txn, MDB_page *mp, int num, int ov, MDB_page **ret)
+mdb_page_unspill(MDB_txn *txn, MDB_page *mp, MDB_page **ret)
 {
 	MDB_env *env = txn->mt_env;
 	const MDB_txn *tx2;
@@ -2736,15 +2745,20 @@ mdb_page_unspill(MDB_txn *txn, MDB_page *mp, int num, int ov, MDB_page **ret)
 		x = mdb_midl_search(tx2->mt_spill_pgs, pn);
 		if (x <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[x] == pn) {
 			MDB_page *np;
+			int num;
 			if (txn->mt_dirty_room == 0)
 				return MDB_TXN_FULL;
+			if (IS_OVERFLOW(mp))
+				num = mp->mp_pages;
+			else
+				num = 1;
 			if (env->me_flags & MDB_WRITEMAP) {
 				np = mp;
 			} else {
 				np = mdb_page_malloc(txn, num, 1);
 				if (!np)
 					return ENOMEM;
-				if (ov)
+				if (num > 1)
 					memcpy(np, mp, num * env->me_psize);
 				else
 					mdb_page_copy(np, mp, env->me_psize);
@@ -2762,9 +2776,8 @@ mdb_page_unspill(MDB_txn *txn, MDB_page *mp, int num, int ov, MDB_page **ret)
 				 * page remains spilled until child commits
 				 */
 
-			mdb_page_dirty(txn, np, ov);
-			if (!ov)
-				np->mp_flags |= P_DIRTY;
+			mdb_page_dirty(txn, np);
+			np->mp_flags |= P_DIRTY;
 			*ret = np;
 			break;
 		}
@@ -2789,14 +2802,14 @@ mdb_page_touch(MDB_cursor *mc)
 	if (!F_ISSET(mp->mp_flags, P_DIRTY)) {
 		if (txn->mt_flags & MDB_TXN_SPILLS) {
 			np = NULL;
-			rc = mdb_page_unspill(txn, mp, 1, 0, &np);
+			rc = mdb_page_unspill(txn, mp, &np);
 			if (rc)
 				goto fail;
 			if (np)
 				goto done;
 		}
 		if ((rc = mdb_midl_need(&txn->mt_free_pgs, 1)) ||
-			(rc = mdb_page_alloc(mc, 1, 0, &np)))
+			(rc = mdb_page_alloc(mc, 1, &np)))
 			goto fail;
 		pgno = np->mp_pgno;
 		DPRINTF(("touched db %d page %"Yu" -> %"Yu, DDBI(mc),
@@ -3148,7 +3161,9 @@ mdb_txn_renew0(MDB_txn *txn)
 		txn->mt_free_pgs = env->me_free_pgs;
 		txn->mt_free_pgs[0] = 0;
 		txn->mt_spill_pgs = NULL;
+#if OVERFLOW_NOTYET
 		txn->mt_dirty_ovs = NULL;
+#endif
 		env->me_txn = txn;
 		memcpy(txn->mt_dbiseqs, env->me_dbiseqs, env->me_maxdbs * sizeof(unsigned int));
 	}
@@ -3273,7 +3288,9 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 		txn->mt_dirty_room = parent->mt_dirty_room;
 		txn->mt_u.dirty_list[0].mid = 0;
 		txn->mt_spill_pgs = NULL;
+#if OVERFLOW_NOTYET
 		txn->mt_dirty_ovs = NULL;
+#endif
 		txn->mt_next_pgno = parent->mt_next_pgno;
 		parent->mt_flags |= MDB_TXN_HAS_CHILD;
 		parent->mt_child = txn;
@@ -3434,7 +3451,9 @@ mdb_txn_end(MDB_txn *txn, unsigned mode)
 			env->me_pgstate = ((MDB_ntxn *)txn)->mnt_pgstate;
 			mdb_midl_free(txn->mt_free_pgs);
 			mdb_midl_free(txn->mt_spill_pgs);
+#if OVERFLOW_NOTYET
 			mdb_mid2l_free(txn->mt_dirty_ovs);
+#endif
 			free(txn->mt_u.dirty_list);
 		}
 
@@ -7981,7 +8000,7 @@ prep_subDB:
 					dummy.md_entries = NUMKEYS(fp);
 					xdata.mv_size = sizeof(MDB_db);
 					xdata.mv_data = &dummy;
-					if ((rc = mdb_page_alloc(mc, 1, 0, &mp)))
+					if ((rc = mdb_page_alloc(mc, 1, &mp)))
 						return rc;
 					offset = env->me_psize - olddata.mv_size;
 					flags |= F_DUPDATA|F_SUBDATA;
@@ -8031,13 +8050,13 @@ current:
 			  if (!(omp->mp_flags & P_DIRTY) &&
 				  (level || (env->me_flags & MDB_WRITEMAP)))
 			  {
-				rc = mdb_page_unspill(mc->mc_txn, omp, ovpages, 1, &omp);
+				rc = mdb_page_unspill(mc->mc_txn, omp, &omp);
 				if (rc)
 					return rc;
 				level = 0;		/* dirty in this txn or clean */
 			  }
 			  /* Is it dirty? */
-			  if (ovp.op_txnid == mc->mc_txn->mt_txnid) {
+			  if (omp->mp_flags & P_DIRTY) {
 				/* yes, overwrite it. Note in this case we don't
 				 * bother to try shrinking the page if the new data
 				 * is smaller than the overflow threshold.
@@ -8063,10 +8082,12 @@ current:
 						 * Copy end of page, adjusting alignment so
 						 * compiler may copy words instead of bytes.
 						 */
-						off = data->mv_size & -sizeof(size_t);
+						off = (PAGEHDRSZ + data->mv_size) & -sizeof(size_t);
 						memcpy((size_t *)((char *)np + off),
 							(size_t *)((char *)omp + off), sz - off);
+						sz = PAGEHDRSZ;
 					}
+					memcpy(np, omp, sz); /* Copy beginning of page */
 					omp = np;
 				}
 				SETDSZ(leaf, data->mv_size);
@@ -8351,7 +8372,7 @@ mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp)
 	MDB_page	*np;
 	int rc;
 
-	if ((rc = mdb_page_alloc(mc, num, flags & P_OVERFLOW, &np)))
+	if ((rc = mdb_page_alloc(mc, num, &np)))
 		return rc;
 	DPRINTF(("allocated new mpage %"Yu", page size %u",
 	    np->mp_pgno, mc->mc_txn->mt_env->me_psize));
@@ -8540,7 +8561,9 @@ update:
 			else
 				memcpy(ndata, data->mv_data, data->mv_size);
 		} else {
-			ndata = ((MDB_dovpage *)ofp)->mp_ptr;
+			MDB_ovpage ovp = {ofp->mp_pgno, mc->mc_txn->mt_txnid, ofp->mp_pages};
+			memcpy(ndata, &ovp, sizeof(MDB_ovpage));
+			ndata = METADATA(ofp);
 			if (F_ISSET(flags, MDB_RESERVE))
 				data->mv_data = ndata;
 			else

From ca8e48d994673bcda4c92ccd79b4c7b2806ac41a Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 25 Jul 2017 20:17:21 +0200
Subject: [PATCH 319/504] Fix last commit - sizeof(pgno_t) and C90 compat

---
 libraries/liblmdb/mdb.c | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index ad5dce6b76..3e4257493a 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1037,7 +1037,7 @@ typedef struct MDB_page {
 	/** Test if a page is a sub page */
 #define IS_SUBP(p)	 F_ISSET((p)->mp_flags, P_SUBP)
 
-	/** Header for overflow pages, stored in an F_BIGDATA node */
+	/** Info about overflow page, stored in an F_BIGDATA node */
 typedef struct MDB_ovpage {
 	pgno_t	op_pgno;
 	txnid_t	op_txnid;
@@ -8412,7 +8412,7 @@ mdb_leaf_size(MDB_env *env, MDB_val *key, MDB_val *data)
 	sz = LEAFSIZE(key, data);
 	if (sz > env->me_nodemax) {
 		/* put on overflow page */
-		sz -= data->mv_size - sizeof(pgno_t);
+		sz -= data->mv_size - sizeof(MDB_ovpage);
 	}
 
 	return EVEN(sz + sizeof(indx_t));
@@ -8437,7 +8437,7 @@ mdb_branch_size(MDB_env *env, MDB_val *key)
 	if (sz > env->me_nodemax) {
 		/* put on overflow page */
 		/* not implemented */
-		/* sz -= key->size - sizeof(pgno_t); */
+		/* sz -= key->size - sizeof(MDB_ovpage); */
 	}
 
 	return sz + sizeof(indx_t);
@@ -8511,7 +8511,7 @@ mdb_node_add(MDB_cursor *mc, indx_t indx,
 			/* Put data on overflow page. */
 			DPRINTF(("data size is %"Z"u, node would be %"Z"u, put data on overflow page",
 			    data->mv_size, node_size+data->mv_size));
-			node_size = EVEN(node_size + sizeof(pgno_t));
+			node_size = EVEN(node_size + sizeof(MDB_ovpage));
 			if ((ssize_t)node_size > room)
 				goto full;
 			if ((rc = mdb_page_new(mc, P_OVERFLOW, ovpages, &ofp)))
@@ -8561,7 +8561,10 @@ update:
 			else
 				memcpy(ndata, data->mv_data, data->mv_size);
 		} else {
-			MDB_ovpage ovp = {ofp->mp_pgno, mc->mc_txn->mt_txnid, ofp->mp_pages};
+			MDB_ovpage ovp;
+			ovp.op_pgno = ofp->mp_pgno;
+			ovp.op_txnid = mc->mc_txn->mt_txnid;
+			ovp.op_pages = ofp->mp_pages;
 			memcpy(ndata, &ovp, sizeof(MDB_ovpage));
 			ndata = METADATA(ofp);
 			if (F_ISSET(flags, MDB_RESERVE))

From 74928f6a3b8e22e1690a962d41d0e9f4a1004f32 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 25 Jul 2017 21:03:48 +0200
Subject: [PATCH 320/504] Fix broken mdb_page_get(ovpage) optimization

---
 libraries/liblmdb/mdb.c | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 3e4257493a..9da57c6d49 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -8341,10 +8341,7 @@ mdb_cursor_del(MDB_cursor *mc, unsigned int flags)
 		MDB_ovpage ovp;
 
 		memcpy(&ovp, NODEDATA(leaf), sizeof(ovp));
-		/* note we don't care about page count here since
-		 * we're just adding pgno to the freelist anyway
-		 */
-		if ((rc = MDB_PAGE_GET(mc, ovp.op_pgno, 1, &omp, NULL)) ||
+		if ((rc = MDB_PAGE_GET(mc, ovp.op_pgno, ovp.op_pages, &omp, NULL)) ||
 			(rc = mdb_ovpage_free(mc, omp)))
 			goto fail;
 	}

From 214aff2359711c45d9c9cf27828d836c815d8c71 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 25 Jul 2017 21:07:14 +0200
Subject: [PATCH 321/504] Move misplaced OVERFLOW_NOTYET code

...so it would set pgno,txnid in the MDB_dovpage, not the actual ovpage
---
 libraries/liblmdb/mdb.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 9da57c6d49..5857a3c661 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1044,11 +1044,13 @@ typedef struct MDB_ovpage {
 	mdb_size_t	op_pages;
 } MDB_ovpage;
 
+#if OVERFLOW_NOTYET
 	/** Header for a dirty overflow page in memory */
 typedef struct MDB_dovpage {
 	MDB_page_header mp_hdr;
 	void	*mp_ptr;
 } MDB_dovpage;
+#endif
 
 	/** The number of overflow pages needed to store the given size. */
 #define OVPAGES(size, psize)	((PAGEHDRSZ-1 + (size)) / (psize) + 1)
@@ -2674,14 +2676,16 @@ search_done:
 	} else {
 		txn->mt_next_pgno = pgno + num;
 	}
-	np->mp_pgno = pgno;
-	np->mp_txnid = txn->mt_txnid;
 #if OVERFLOW_NOTYET
 	if (ov) {
 		dph->mp_hdr = np->mp_hdr;
 		dph->mp_ptr = np;
 		np = (MDB_page *)dph;
 	}
+#endif
+	np->mp_pgno = pgno;
+	np->mp_txnid = txn->mt_txnid;
+#if OVERFLOW_NOTYET
 	mdb_page_dirty(txn, np, ov);
 #else
 	mdb_page_dirty(txn, np);

From 9e3603eb9db7551ef112b472064b2a0ad2d7ea74 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 25 Jul 2017 21:27:36 +0200
Subject: [PATCH 322/504] Fix mdb_enctest()

Accept a partial ovpage.  I.e. decryption of the beginning
should not depend on the data at the end.
Make the key and IV less regular. (Divisor 67 has period>64.)
---
 libraries/liblmdb/mdb.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 5857a3c661..9d6467b3a4 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5549,9 +5549,9 @@ mdb_enctest(const MDB_val *src, MDB_val *dst, const MDB_val *key, int encdec)
 {
 	mdb_size_t *sptr = src->mv_data, *dptr = dst->mv_data;
 	mdb_size_t x=*(mdb_size_t*)key[0].mv_data, v=*(mdb_size_t*)key[1].mv_data;
-	int i = dst->mv_size / sizeof(mdb_size_t);
+	int i, len = dst->mv_size / sizeof(mdb_size_t);
 
-	while (--i >= 0)
+	for (i = 0; i < len; i++)
 		x += v += i + sptr[i] + (dptr[i] = sptr[i] ^ x);
 }
 #endif	/* MDB_RPAGE_CACHE */
@@ -5581,8 +5581,8 @@ mdb_env_getflags(MDB_env *env)
 		if (flags & MDB_ENCRYPT) {
 #if MDB_RPAGE_CACHE
 			if (!env->me_encfunc) {
-				static mdb_size_t k = MDB_SIZE_MAX/9*37;
-				mdb_size_t iv = ((mdb_size_t)env ^ env->me_pid);
+				static mdb_size_t k = (MDB_SIZE_MAX/67*73) | 1;
+				mdb_size_t iv = ((mdb_size_t)env ^ env->me_pid) * k;
 				MDB_val keys[2] = { {sizeof(k), &k}, {sizeof(iv), NULL} };
 				int rc;
 				keys[1].mv_data = &iv;

From 4b7e430303b281f8ddc187215aee9f552328f70b Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 25 Jul 2017 21:27:36 +0200
Subject: [PATCH 323/504] mdb_page_touch: Always set MDB_TXN_ERROR on error

---
 libraries/liblmdb/mdb.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 9d6467b3a4..e2d3b53998 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2848,8 +2848,10 @@ mdb_page_touch(MDB_cursor *mc)
 		mdb_cassert(mc, dl[0].mid < MDB_IDL_UM_MAX);
 		/* No - copy it */
 		np = mdb_page_malloc(txn, 1, 1);
-		if (!np)
-			return ENOMEM;
+		if (!np) {
+			rc = ENOMEM;
+			goto fail;
+		}
 		mid.mid = pgno;
 		mid.mptr = np;
 		rc = mdb_mid2l_insert(dl, &mid);

From 0e17ba43a818f6bdab7759586e247bae12692c25 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 25 Jul 2017 21:27:36 +0200
Subject: [PATCH 324/504] Verify final loose_count in mdb_txn_commit()

Mismatch may indicate that pages leaked or got used twice
in the same snapshot.
---
 libraries/liblmdb/mdb.c | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index e2d3b53998..3e0aa98340 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -3544,7 +3544,7 @@ mdb_freelist_save(MDB_txn *txn)
 	 */
 	MDB_cursor mc;
 	MDB_env	*env = txn->mt_env;
-	int rc, maxfree_1pg = env->me_maxfree_1pg, more = 1;
+	int rc, maxfree_1pg = env->me_maxfree_1pg, more = 1, lost_loose = 0;
 	txnid_t	pglast = 0, head_id = 0;
 	pgno_t	freecnt = 0, *free_pgs, *mop;
 	ssize_t	head_room = 0, total_room = 0, mop_len, clean_limit;
@@ -3565,6 +3565,7 @@ mdb_freelist_save(MDB_txn *txn)
 		MDB_page *mp = txn->mt_loose_pgs;
 		if ((rc = mdb_midl_need(&txn->mt_free_pgs, txn->mt_loose_count)) != 0)
 			return rc;
+		lost_loose = txn->mt_loose_count;
 		for (; mp; mp = NEXT_LOOSE_PAGE(mp))
 			mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno);
 		txn->mt_loose_pgs = NULL;
@@ -3682,6 +3683,7 @@ mdb_freelist_save(MDB_txn *txn)
 		/* Room for loose pages + temp IDL with same */
 		if ((rc = mdb_midl_need(&env->me_pghead, 2*count+1)) != 0)
 			return rc;
+		lost_loose += count;
 		mop = env->me_pghead;
 		loose = mop + MDB_IDL_ALLOCLEN(mop) - count;
 		for (count = 0; mp; mp = NEXT_LOOSE_PAGE(mp))
@@ -3721,6 +3723,11 @@ mdb_freelist_save(MDB_txn *txn)
 				break;
 		}
 	}
+
+	/* Restore this so we can check vs. dirty_list after mdb_page_flush() */
+	if (! (txn->mt_flags & MDB_TXN_WRITEMAP))
+		txn->mt_loose_count += lost_loose;
+
 	return rc;
 }
 
@@ -4156,6 +4163,10 @@ mdb_txn_commit(MDB_txn *txn)
 
 	if ((rc = mdb_page_flush(txn, 0)))
 		goto fail;
+	if ((unsigned)txn->mt_loose_count != txn->mt_u.dirty_list[0].mid) {
+		rc = MDB_PROBLEM; /* mt_loose_pgs does not match dirty_list */
+		goto fail;
+	}
 	if (!F_ISSET(txn->mt_flags, MDB_TXN_NOSYNC) &&
 		(rc = mdb_env_sync0(env, 0, txn->mt_next_pgno)))
 		goto fail;

From 12ee1a2d7104616ccb812db5d5f3548da507d9fc Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 25 Jul 2017 21:27:36 +0200
Subject: [PATCH 325/504] Use mdb_page_loose() more

---
 libraries/liblmdb/mdb.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 3e0aa98340..05273a25ab 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -9493,7 +9493,7 @@ mdb_rebalance(MDB_cursor *mc)
 			mc->mc_db->md_root = P_INVALID;
 			mc->mc_db->md_depth = 0;
 			mc->mc_db->md_leaf_pages = 0;
-			rc = mdb_midl_append(&mc->mc_txn->mt_free_pgs, mp->mp_pgno);
+			rc = mdb_page_loose(mc, mp);
 			if (rc)
 				return rc;
 			/* Adjust cursors pointing to mp */
@@ -9521,7 +9521,7 @@ mdb_rebalance(MDB_cursor *mc)
 		} else if (IS_BRANCH(mp) && NUMKEYS(mp) == 1) {
 			int i;
 			DPUTS("collapsing root page!");
-			rc = mdb_midl_append(&mc->mc_txn->mt_free_pgs, mp->mp_pgno);
+			rc = mdb_page_loose(mc, mp);
 			if (rc)
 				return rc;
 			mc->mc_db->md_root = NODEPGNO(NODEPTR(mp, 0));

From 6dd5a4d7f6ba65047508a9de808b8e8adddd8d39 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 25 Jul 2017 21:27:36 +0200
Subject: [PATCH 326/504] Protect freelist at end of mdb_freelist_save()

---
 libraries/liblmdb/mdb.c | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 05273a25ab..8229b1fdf9 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -3696,11 +3696,17 @@ mdb_freelist_save(MDB_txn *txn)
 		mop_len = mop[0];
 	}
 
-	/* Fill in the reserved me_pghead records */
+	/* Fill in the reserved me_pghead records.  Everything is finally
+	 * in place, so this will not allocate or free any DB pages.
+	 */
 	rc = MDB_SUCCESS;
 	if (mop_len) {
 		MDB_val key, data;
 
+		/* Protect DB env from any (buggy) freelist use when saving mop */
+		env->me_pghead = NULL;
+		txn->mt_dirty_room = 0;
+
 		mop += mop_len;
 		rc = mdb_cursor_first(&mc, &key, &data);
 		for (; !rc; rc = mdb_cursor_next(&mc, &key, &data, MDB_NEXT)) {
@@ -3714,14 +3720,17 @@ mdb_freelist_save(MDB_txn *txn)
 				len = mop_len;
 				data.mv_size = (len + 1) * sizeof(MDB_ID);
 			}
+			mop_len -= len;
 			data.mv_data = mop -= len;
 			save = mop[0];
 			mop[0] = len;
 			rc = mdb_cursor_put(&mc, &key, &data, MDB_CURRENT);
 			mop[0] = save;
-			if (rc || !(mop_len -= len))
+			if (rc || !mop_len)
 				break;
 		}
+
+		env->me_pghead = mop - mop_len;
 	}
 
 	/* Restore this so we can check vs. dirty_list after mdb_page_flush() */

From 34461da97b4dc43394844a2299e9452c07cfd8f2 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 25 Jul 2017 21:27:36 +0200
Subject: [PATCH 327/504] Skip unnecessary mdb_page_get()s when spilling

Do not bring in pages merely to see if they should be skipped.
---
 libraries/liblmdb/mdb.c | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 8229b1fdf9..2076990fa7 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2237,8 +2237,8 @@ mdb_pages_xkeep(MDB_cursor *mc, unsigned pflags, int all)
 	MDB_xcursor *mx;
 	MDB_page *dp, *mp;
 	MDB_node *leaf;
-	unsigned i, j;
-	int rc = MDB_SUCCESS, level;
+	unsigned i, j, x;
+	int rc = MDB_SUCCESS;
 
 	/* Mark pages seen by cursors: First m0, then tracked cursors */
 	for (i = txn->mt_numdbs;; ) {
@@ -2270,14 +2270,17 @@ mdb_pages_xkeep(MDB_cursor *mc, unsigned pflags, int all)
 mark_done:
 	if (all) {
 		/* Mark dirty root pages */
+		MDB_ID2L dl = txn->mt_u.dirty_list;
 		for (i=0; i<txn->mt_numdbs; i++) {
 			if (txn->mt_dbflags[i] & DB_DIRTY) {
 				pgno_t pgno = txn->mt_dbs[i].md_root;
 				if (pgno == P_INVALID)
 					continue;
-				if ((rc = MDB_PAGE_GET(m0, pgno, 1, &dp, &level)) != MDB_SUCCESS)
-					break;
-				if ((dp->mp_flags & Mask) == pflags && level <= 1)
+				x = mdb_mid2l_search(dl, pgno);
+				if (! (x <= dl[0].mid && dl[x].mid == pgno))
+					continue;
+				dp = dl[x].mptr;
+				if ((dp->mp_flags & Mask) == pflags)
 					dp->mp_flags ^= P_KEEP;
 			}
 		}

From 3b55853fe0563bf1f49c84bf1a49e6b324472d5f Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 25 Jul 2017 21:27:36 +0200
Subject: [PATCH 328/504] Drop unnecessary searches in the spill lists

Check with IS_MUTABLE() if an MDB_page is spilled, instead of
searching spill lists.  When unspilling, skip parent spill lists.
---
 libraries/liblmdb/mdb.c | 80 ++++++++++++++++++++---------------------
 1 file changed, 38 insertions(+), 42 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 2076990fa7..acab891d38 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1037,6 +1037,9 @@ typedef struct MDB_page {
 	/** Test if a page is a sub page */
 #define IS_SUBP(p)	 F_ISSET((p)->mp_flags, P_SUBP)
 
+	/** Test if this non-sub page belongs to the current snapshot */
+#define IS_MUTABLE(txn, p)	((p)->mp_txnid == (txn)->mt_txnid)
+
 	/** Info about overflow page, stored in an F_BIGDATA node */
 typedef struct MDB_ovpage {
 	pgno_t	op_pgno;
@@ -2730,31 +2733,34 @@ mdb_page_copy(MDB_page *dst, MDB_page *src, unsigned int psize)
 	}
 }
 
-/** Pull a page off the txn's spill list, if present.
- * If a page being referenced was spilled to disk in this txn, bring
- * it back and make it dirty/writable again.
+/** Bring back a page which this txn spilled to disk; make it writable again.
  * @param[in] txn the transaction handle.
- * @param[in] mp the page being referenced. It must not be dirty.
- * @param[out] ret the writable page, if any. ret is unchanged if
- * mp wasn't spilled.
+ * @param[in] mp the spilled page.
+ * @param[out] ret the writable page.
  */
 static int
 mdb_page_unspill(MDB_txn *txn, MDB_page *mp, MDB_page **ret)
 {
 	MDB_env *env = txn->mt_env;
-	const MDB_txn *tx2;
 	unsigned x;
 	pgno_t pgno = mp->mp_pgno, pn = pgno << 1;
 
-	for (tx2 = txn; tx2; tx2=tx2->mt_parent) {
-		if (!tx2->mt_spill_pgs)
-			continue;
-		x = mdb_midl_search(tx2->mt_spill_pgs, pn);
-		if (x <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[x] == pn) {
+	if (txn->mt_dirty_room == 0)
+		return MDB_TXN_FULL;
+
+	/* x = position in current spill list, or 0 */
+	x = 0;
+	if (txn->mt_spill_pgs) {
+		x = mdb_midl_search(txn->mt_spill_pgs, pn);
+		if (! (x <= txn->mt_spill_pgs[0] && txn->mt_spill_pgs[x] == pn))
+			x = 0;
+	}
+	if (x == 0 && !txn->mt_parent)
+		return MDB_PROBLEM;		/* should be a spilled page */
+
+	{
 			MDB_page *np;
 			int num;
-			if (txn->mt_dirty_room == 0)
-				return MDB_TXN_FULL;
 			if (IS_OVERFLOW(mp))
 				num = mp->mp_pages;
 			else
@@ -2770,7 +2776,7 @@ mdb_page_unspill(MDB_txn *txn, MDB_page *mp, MDB_page **ret)
 				else
 					mdb_page_copy(np, mp, env->me_psize);
 			}
-			if (tx2 == txn) {
+			if (x) {
 				/* If in current txn, this page is no longer spilled.
 				 * If it happens to be the last page, truncate the spill list.
 				 * Otherwise mark it as deleted by setting the LSB.
@@ -2786,10 +2792,8 @@ mdb_page_unspill(MDB_txn *txn, MDB_page *mp, MDB_page **ret)
 			mdb_page_dirty(txn, np);
 			np->mp_flags |= P_DIRTY;
 			*ret = np;
-			break;
-		}
+			return MDB_SUCCESS;
 	}
-	return MDB_SUCCESS;
 }
 
 /** Touch a page: make it dirty and re-insert into tree with updated pgno.
@@ -2807,13 +2811,11 @@ mdb_page_touch(MDB_cursor *mc)
 	int rc;
 
 	if (!F_ISSET(mp->mp_flags, P_DIRTY)) {
-		if (txn->mt_flags & MDB_TXN_SPILLS) {
-			np = NULL;
+		if (IS_MUTABLE(txn, mp)) {
 			rc = mdb_page_unspill(txn, mp, &np);
 			if (rc)
 				goto fail;
-			if (np)
-				goto done;
+			goto done;
 		}
 		if ((rc = mdb_midl_need(&txn->mt_free_pgs, 1)) ||
 			(rc = mdb_page_alloc(mc, 1, &np)))
@@ -6864,7 +6866,6 @@ mdb_ovpage_free(MDB_cursor *mc, MDB_page *mp)
 	pgno_t pg = mp->mp_pgno;
 	unsigned x = 0, ovpages = mp->mp_pages;
 	MDB_env *env = txn->mt_env;
-	MDB_IDL sl = txn->mt_spill_pgs;
 	MDB_ID pn = pg << 1;
 	int rc;
 
@@ -6877,11 +6878,7 @@ mdb_ovpage_free(MDB_cursor *mc, MDB_page *mp)
 	 * Unsupported in nested txns: They would need to hide the page
 	 * range in ancestor txns' dirty and spilled lists.
 	 */
-	if (env->me_pghead &&
-		!txn->mt_parent &&
-		((mp->mp_flags & P_DIRTY) ||
-		 (sl && (x = mdb_midl_search(sl, pn)) <= sl[0] && sl[x] == pn)))
-	{
+	if (IS_MUTABLE(txn, mp) && env->me_pghead && !txn->mt_parent) {
 		unsigned i, j;
 		pgno_t *mop;
 		MDB_ID2 *dl, ix, iy;
@@ -6889,6 +6886,11 @@ mdb_ovpage_free(MDB_cursor *mc, MDB_page *mp)
 		if (rc)
 			return rc;
 		if (!(mp->mp_flags & P_DIRTY)) {
+			MDB_IDL sl = txn->mt_spill_pgs;
+			if (sl)
+				x = mdb_midl_search(sl, pn);
+			if (! (sl && x <= sl[0] && sl[x] == pn))
+				return MDB_PROBLEM;
 			/* This page is no longer spilled */
 			if (x == sl[0])
 				sl[0]--;
@@ -8073,23 +8075,18 @@ current:
 				return rc2;
 			ovpages = ovp.op_pages;
 
-			/* Is the ov page large enough? */
-			if (ovpages >= dpages) {
-			  /* Did we dirty it in this txn? */
-			  if (!(omp->mp_flags & P_DIRTY) &&
-				  (level || (env->me_flags & MDB_WRITEMAP)))
-			  {
-				rc = mdb_page_unspill(mc->mc_txn, omp, &omp);
-				if (rc)
-					return rc;
-				level = 0;		/* dirty in this txn or clean */
-			  }
-			  /* Is it dirty? */
-			  if (omp->mp_flags & P_DIRTY) {
+			/* Is the ov page big enough and from this txn (or a parent)? */
+			if (ovpages >= dpages && IS_MUTABLE(mc->mc_txn, omp)) {
 				/* yes, overwrite it. Note in this case we don't
 				 * bother to try shrinking the page if the new data
 				 * is smaller than the overflow threshold.
 				 */
+				if (!(omp->mp_flags & P_DIRTY)) {
+					rc = mdb_page_unspill(mc->mc_txn, omp, &omp);
+					if (rc)
+						return rc;
+					level = 0;		/* dirty in this txn */
+				}
 				if (level > 1) {
 					/* It is writable only in a parent txn */
 					size_t sz = (size_t) env->me_psize * ovpages, off;
@@ -8125,7 +8122,6 @@ current:
 				else
 					memcpy(METADATA(omp), data->mv_data, data->mv_size);
 				return MDB_SUCCESS;
-			  }
 			}
 			if ((rc2 = mdb_ovpage_free(mc, omp)) != MDB_SUCCESS)
 				return rc2;

From 0a99df54afed3caedbc18668e2e534a1c20d1ac8 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 25 Jul 2017 21:27:36 +0200
Subject: [PATCH 329/504] Divide page flags in P_ADM_FLAGS and the rest

P_ADM_FLAGS flags are tied to the page, even through page_loose-alloc.
OTOH mdb_page_split() should only duplicate the relevant flags.

This is just the code for the feature, P_ADM_FLAGS is 0 for now.
We'll need it for P_HIDESPILL later.
---
 libraries/liblmdb/mdb.c | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index acab891d38..55a2ded3f2 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -979,6 +979,8 @@ typedef struct MDB_page_header {
 #define	P_DIRTY_OVF	 0x2000		/**< page has dirty overflow nodes */
 #define	P_LOOSE		 0x4000		/**< page was dirtied then freed, can be reused */
 #define	P_KEEP		 0x8000		/**< leave this page alone during spill */
+/** Persistent flags for page administration rather than page contents */
+#define	P_ADM_FLAGS	 0 /* later... */
 /** @} */
 	uint16_t	mh_flags;		/**< @ref mdb_page */
 #define mp_lower	mp_pb.pb.pb_lower
@@ -2054,6 +2056,7 @@ init:
 	} else {
 		txn->mt_flags |= MDB_TXN_ERROR;
 	}
+	ret->mp_flags = 0;
 	return ret;
 }
 /** Free a single page.
@@ -2532,6 +2535,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
 		}
 #endif
 		*mp = np;
+		np->mp_flags &= P_ADM_FLAGS;
 		return MDB_SUCCESS;
 	}
 
@@ -2691,6 +2695,7 @@ search_done:
 #endif
 	np->mp_pgno = pgno;
 	np->mp_txnid = txn->mt_txnid;
+	np->mp_flags = 0;
 #if OVERFLOW_NOTYET
 	mdb_page_dirty(txn, np, ov);
 #else
@@ -2807,6 +2812,7 @@ mdb_page_touch(MDB_cursor *mc)
 	MDB_page *mp = mc->mc_pg[mc->mc_top], *np;
 	MDB_txn *txn = mc->mc_txn;
 	MDB_cursor *m2, *m3;
+	unsigned np_flags;
 	pgno_t	pgno;
 	int rc;
 
@@ -2865,9 +2871,10 @@ mdb_page_touch(MDB_cursor *mc)
 		return 0;
 	}
 
+	np_flags = np->mp_flags;	/* P_ADM_FLAGS */
 	mdb_page_copy(np, mp, txn->mt_env->me_psize);
+	np->mp_flags |= np_flags | P_DIRTY;
 	np->mp_pgno = pgno;
-	np->mp_flags |= P_DIRTY;
 
 done:
 	np->mp_txnid = txn->mt_txnid;
@@ -8033,6 +8040,7 @@ prep_subDB:
 					xdata.mv_data = &dummy;
 					if ((rc = mdb_page_alloc(mc, 1, &mp)))
 						return rc;
+					fp_flags |= mp->mp_flags; /* P_ADM_FLAGS */
 					offset = env->me_psize - olddata.mv_size;
 					flags |= F_DUPDATA|F_SUBDATA;
 					dummy.md_root = mp->mp_pgno;
@@ -8398,7 +8406,7 @@ mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp)
 		return rc;
 	DPRINTF(("allocated new mpage %"Yu", page size %u",
 	    np->mp_pgno, mc->mc_txn->mt_env->me_psize));
-	np->mp_flags = flags | P_DIRTY;
+	np->mp_flags |= flags | P_DIRTY;
 	np->mp_lower = (PAGEHDRSZ-PAGEBASE);
 	np->mp_upper = mc->mc_txn->mt_env->me_psize - PAGEBASE;
 
@@ -9840,7 +9848,8 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 	    DKEY(newkey), mc->mc_ki[mc->mc_top], nkeys));
 
 	/* Create a right sibling. */
-	if ((rc = mdb_page_new(mc, mp->mp_flags, 1, &rp)))
+	rc = mdb_page_new(mc, mp->mp_flags & (P_BRANCH|P_LEAF|P_LEAF2), 1, &rp);
+	if (rc)
 		return rc;
 	rp->mp_pad = mp->mp_pad;
 	DPRINTF(("new right sibling: page %"Yu, rp->mp_pgno));

From c83434e1785a76891ef759f4286e2685532a6a9c Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 25 Jul 2017 21:27:36 +0200
Subject: [PATCH 330/504] Demo - move P_DIRTY to P_ADM_FLAGS, page_dirty()

...to have something substantial to test P_ADM_FLAGS with,
at least until we drop the P_DIRTY flag.
---
 libraries/liblmdb/mdb.c | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 55a2ded3f2..0721d9294b 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -980,7 +980,7 @@ typedef struct MDB_page_header {
 #define	P_LOOSE		 0x4000		/**< page was dirtied then freed, can be reused */
 #define	P_KEEP		 0x8000		/**< leave this page alone during spill */
 /** Persistent flags for page administration rather than page contents */
-#define	P_ADM_FLAGS	 0 /* later... */
+#define	P_ADM_FLAGS	 (P_DIRTY)
 /** @} */
 	uint16_t	mh_flags;		/**< @ref mdb_page */
 #define mp_lower	mp_pb.pb.pb_lower
@@ -2446,13 +2446,15 @@ mdb_find_oldest(MDB_txn *txn)
 	return oldest;
 }
 
-/** Add a page to the txn's dirty list */
+/** Mark a page as dirty and add it to the txn's dirty list */
 static void
 mdb_page_dirty(MDB_txn *txn, MDB_page *mp)
 {
 	MDB_ID2 mid;
 	int rc, (*insert)(MDB_ID2L, MDB_ID2 *);
 
+	mp->mp_flags |= P_DIRTY;
+
 	if (txn->mt_flags & MDB_TXN_WRITEMAP) {
 		insert = mdb_mid2l_append;
 	} else {
@@ -2795,7 +2797,6 @@ mdb_page_unspill(MDB_txn *txn, MDB_page *mp, MDB_page **ret)
 				 */
 
 			mdb_page_dirty(txn, np);
-			np->mp_flags |= P_DIRTY;
 			*ret = np;
 			return MDB_SUCCESS;
 	}
@@ -2867,13 +2868,14 @@ mdb_page_touch(MDB_cursor *mc)
 		mid.mptr = np;
 		rc = mdb_mid2l_insert(dl, &mid);
 		mdb_cassert(mc, rc == 0);
+		np->mp_flags |= P_DIRTY;
 	} else {
 		return 0;
 	}
 
 	np_flags = np->mp_flags;	/* P_ADM_FLAGS */
 	mdb_page_copy(np, mp, txn->mt_env->me_psize);
-	np->mp_flags |= np_flags | P_DIRTY;
+	np->mp_flags |= np_flags;
 	np->mp_pgno = pgno;
 
 done:
@@ -7895,7 +7897,7 @@ mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data,
 			/* Too big for a node, insert in sub-DB.  Set up an empty
 			 * "old sub-page" for prep_subDB to expand to a full page.
 			 */
-			fp_flags = P_LEAF|P_DIRTY;
+			fp_flags = P_LEAF;
 			fp = env->me_pbuf;
 			fp->mp_pad = data->mv_size; /* used if MDB_DUPFIXED */
 			fp->mp_lower = fp->mp_upper = (PAGEHDRSZ-PAGEBASE);
@@ -8007,7 +8009,6 @@ more:
 					}
 					/* FALLTHRU: Big enough MDB_DUPFIXED sub-page */
 				case MDB_CURRENT:
-					fp->mp_flags |= P_DIRTY;
 					COPY_PGNO(fp->mp_pgno, mp->mp_pgno);
 					mc->mc_xcursor->mx_cursor.mc_pg[0] = fp;
 					flags |= F_DUPDATA;
@@ -8047,7 +8048,7 @@ prep_subDB:
 					sub_root = mp;
 			}
 			if (mp != fp) {
-				mp->mp_flags = fp_flags | P_DIRTY;
+				mp->mp_flags = fp_flags;
 				mp->mp_pad   = fp->mp_pad;
 				mp->mp_lower = fp->mp_lower;
 				mp->mp_upper = fp->mp_upper + offset;
@@ -8406,7 +8407,7 @@ mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp)
 		return rc;
 	DPRINTF(("allocated new mpage %"Yu", page size %u",
 	    np->mp_pgno, mc->mc_txn->mt_env->me_psize));
-	np->mp_flags |= flags | P_DIRTY;
+	np->mp_flags |= flags;
 	np->mp_lower = (PAGEHDRSZ-PAGEBASE);
 	np->mp_upper = mc->mc_txn->mt_env->me_psize - PAGEBASE;
 

From f1db84d3322912949c0589cb834dd5d370f6a9e7 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 25 Jul 2017 21:27:36 +0200
Subject: [PATCH 331/504] mp_txnid = page state, avoids searching dirty_list

In nested txns, check mp_txnid to see which txn dirtied the page.
This change will also let us remove the P_DIRTY flag, and keep
some flags in (dirty page).mp_txnid if we should need it.
---
 libraries/liblmdb/mdb.c | 195 ++++++++++++++++++++++++++--------------
 1 file changed, 129 insertions(+), 66 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 0721d9294b..1d051e3ba0 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -962,7 +962,26 @@ typedef struct MDB_page_header {
 		pgno_t		p_pgno;	/**< page number */
 		struct MDB_page *p_next; /**< for in-memory list of freed pages */
 	} mh_p;
-	txnid_t		mh_txnid;		/**< txnid that committed this page, unused in meta pages */
+
+	/** If page is clean: snapshot txnid, dirty: txn workid, metapage: unused.
+	 *
+	 *	The value indicates which snapshot/#MDB_txn the page belongs to.
+	 *	Tested with #IS_MUTABLE(), #IS_WRITABLE(), #IS_DIRTY_NW().
+	 *
+	 *	(clean page).mp_txnid == txnid of creator < txnid of later txns.
+	 *	(dirty page).mp_txnid >= mt_workid of creator txn.
+	 *	(dirty page).mt_txnid  < mt_workid of children of creator txn.
+	 *
+	 *	Thus an #MDB_txn can write to pages with mp_txnid >= txn.mt_workid.
+	 *	A page with smaller mp_txnid is dirty in an ancestor txn or clean.
+	 *
+	 *	txn.mt_workid > txn.mt_txnid, to tell apart spilled and dirty pages.
+	 *
+	 *	Finally, ((dirty page).mp_txnid & #MDB_PGTXNID_FLAGMASK) can be used
+	 *	for flags with non-WRITEMAP; it keeps low bits in workid = 0.
+	 */
+	txnid_t		mh_txnid;
+
 	uint16_t	mh_pad;			/**< key size if this is a LEAF2 page */
 /**	@defgroup mdb_page	Page Flags
  *	@ingroup internal
@@ -1040,7 +1059,9 @@ typedef struct MDB_page {
 #define IS_SUBP(p)	 F_ISSET((p)->mp_flags, P_SUBP)
 
 	/** Test if this non-sub page belongs to the current snapshot */
-#define IS_MUTABLE(txn, p)	((p)->mp_txnid == (txn)->mt_txnid)
+#define IS_MUTABLE(txn, p)	((p)->mp_txnid >= (txn)->mt_txnid)
+	/** Test if this non-sub page is writable in this txn (not an ancestor) */
+#define IS_WRITABLE(txn, p)	((p)->mp_txnid >= (txn)->mt_workid)
 
 	/** Info about overflow page, stored in an F_BIGDATA node */
 typedef struct MDB_ovpage {
@@ -1065,6 +1086,27 @@ typedef struct MDB_dovpage {
 	 */
 #define NEXT_LOOSE_PAGE(p)		(*(MDB_page **)((p) + 2))
 
+	/** Mark the page as writable by this txn */
+#ifndef MDB_TEST
+#define SET_PGTXNID(txn, mp)	((mp)->mp_txnid = (txn)->mt_workid)
+#else
+#define SET_PGTXNID(txn, mp) \
+	((mp)->mp_txnid = (txn)->mt_workid \
+	 /* random unused "flags" added when not WRITEMAP for debugging */ \
+	 | (((txn)->mt_flags & MDB_TXN_WRITEMAP) ? 0 : \
+		(MDB_RAND((size_t)(txn)) >> (32-MDB_PGTXNID_FLAGBITS))))
+#define MDB_RAND(x) (mdb_rnd = (mdb_rnd + (unsigned)(x)) * 987654321 + 54321)
+static volatile unsigned mdb_rnd;
+#endif
+
+	/** mp_txnid bits reserved in dirty pages for flags.
+	 *	TODO: For future code with header-free ovpages, if we omit mp_flags
+	 *	from the "header" kept elsewhere.  Otherwise, drop this code.
+	 */
+#define MDB_PGTXNID_FLAGBITS	4
+#define MDB_PGTXNID_STEP		((txnid_t)1 << MDB_PGTXNID_FLAGBITS)
+#define MDB_PGTXNID_FLAGMASK	(MDB_PGTXNID_STEP-1)
+
 	/** Header for a single key/data pair within a page.
 	 * Used in pages of type #P_BRANCH and #P_LEAF without #P_LEAF2.
 	 * We guarantee 2-byte alignment for 'MDB_node's.
@@ -1286,6 +1328,23 @@ struct MDB_txn {
 	 *	aborts, the ID may be re-used by the next writer.
 	 */
 	txnid_t		mt_txnid;
+
+	/** Written to mp_txnid of dirty pages, to be fixed by #mdb_page_flush().
+	 *
+	 *	Value >= 1 + (parent ? parent.last_workid : txnid).
+	 *	See #MDB_page.%mp_txnid.
+	 *
+	 *	An MDB_txn can write to a page when page.mp_txnid >= txn.mt_workid.
+	 *	New children get bigger workid than pages dirty in their parent
+	 *	(i.e. bigger than parent.mt_last_workid).  When children commit,
+	 *	they copy #mt_last_workid to the parent to match their pages.
+	 */
+	txnid_t		mt_workid;
+	/**	Current max mp_txnid of the MDB_txn's dirty pages: Starts as
+	 *	#mt_workid, then grows as it is copied from children who commit.
+	 */
+	txnid_t		mt_last_workid;
+
 	MDB_env		*mt_env;		/**< the DB environment */
 	/** The list of pages that became unused during this transaction.
 	 */
@@ -2185,46 +2244,21 @@ mdb_cursor_unref(MDB_cursor *mc)
 static int
 mdb_page_loose(MDB_cursor *mc, MDB_page *mp)
 {
-	int loose = 0;
-	pgno_t pgno = mp->mp_pgno;
+	int rc = MDB_SUCCESS;
 	MDB_txn *txn = mc->mc_txn;
 
-	if ((mp->mp_flags & P_DIRTY) && mc->mc_dbi != FREE_DBI) {
-		if (txn->mt_parent) {
-			MDB_ID2 *dl = txn->mt_u.dirty_list;
-			/* If txn has a parent, make sure the page is in our
-			 * dirty list.
-			 */
-			if (dl[0].mid) {
-				unsigned x = mdb_mid2l_search(dl, pgno);
-				if (x <= dl[0].mid && dl[x].mid == pgno) {
-					if (mp != dl[x].mptr) { /* bad cursor? */
-						mc->mc_flags &= ~(C_INITIALIZED|C_EOF);
-						txn->mt_flags |= MDB_TXN_ERROR;
-						return MDB_PROBLEM;
-					}
-					/* ok, it's ours */
-					loose = 1;
-				}
-			}
-		} else {
-			/* no parent txn, so it's just ours */
-			loose = 1;
-		}
-	}
-	if (loose) {
+	if (IS_WRITABLE(txn, mp) && mc->mc_dbi != FREE_DBI) {
+		/* Page is dirty in this txn, and is not in freeDB */
 		DPRINTF(("loosen db %d page %"Yu, DDBI(mc), mp->mp_pgno));
 		NEXT_LOOSE_PAGE(mp) = txn->mt_loose_pgs;
 		txn->mt_loose_pgs = mp;
 		txn->mt_loose_count++;
 		mp->mp_flags |= P_LOOSE;
 	} else {
-		int rc = mdb_midl_append(&txn->mt_free_pgs, pgno);
-		if (rc)
-			return rc;
+		rc = mdb_midl_append(&txn->mt_free_pgs, mp->mp_pgno);
 	}
 
-	return MDB_SUCCESS;
+	return rc;
 }
 
 /** Set or clear P_KEEP in dirty, non-overflow, non-sub pages watched by txn.
@@ -2696,7 +2730,7 @@ search_done:
 	}
 #endif
 	np->mp_pgno = pgno;
-	np->mp_txnid = txn->mt_txnid;
+	SET_PGTXNID(txn, np);
 	np->mp_flags = 0;
 #if OVERFLOW_NOTYET
 	mdb_page_dirty(txn, np, ov);
@@ -2797,6 +2831,7 @@ mdb_page_unspill(MDB_txn *txn, MDB_page *mp, MDB_page **ret)
 				 */
 
 			mdb_page_dirty(txn, np);
+			SET_PGTXNID(txn, np);
 			*ret = np;
 			return MDB_SUCCESS;
 	}
@@ -2817,13 +2852,11 @@ mdb_page_touch(MDB_cursor *mc)
 	pgno_t	pgno;
 	int rc;
 
-	if (!F_ISSET(mp->mp_flags, P_DIRTY)) {
-		if (IS_MUTABLE(txn, mp)) {
-			rc = mdb_page_unspill(txn, mp, &np);
-			if (rc)
-				goto fail;
-			goto done;
-		}
+	if (IS_SUBP(mp) || IS_WRITABLE(txn, mp))
+		return MDB_SUCCESS;
+
+	if (!IS_MUTABLE(txn, mp)) {
+		/* Page from an older snapshot */
 		if ((rc = mdb_midl_need(&txn->mt_free_pgs, 1)) ||
 			(rc = mdb_page_alloc(mc, 1, &np)))
 			goto fail;
@@ -2840,25 +2873,20 @@ mdb_page_touch(MDB_cursor *mc)
 		} else {
 			mc->mc_db->md_root = pgno;
 		}
-	} else if (txn->mt_parent && !IS_SUBP(mp)) {
+	} else if (!F_ISSET(mp->mp_flags, P_DIRTY)) {
+		rc = mdb_page_unspill(txn, mp, &np);
+		if (rc)
+			goto fail;
+		goto done;
+	} else {
+		/* Writable in an ancestor txn */
 		MDB_ID2 mid, *dl = txn->mt_u.dirty_list;
 		pgno = mp->mp_pgno;
-		/* If txn has a parent, make sure the page is in our
-		 * dirty list.
-		 */
-		if (dl[0].mid) {
-			unsigned x = mdb_mid2l_search(dl, pgno);
-			if (x <= dl[0].mid && dl[x].mid == pgno) {
-				if (mp != dl[x].mptr) { /* bad cursor? */
-					mc->mc_flags &= ~(C_INITIALIZED|C_EOF);
-					txn->mt_flags |= MDB_TXN_ERROR;
-					return MDB_PROBLEM;
-				}
-				return 0;
-			}
+		if (!txn->mt_parent) {
+			rc = MDB_PROBLEM;
+			goto fail;
 		}
 		mdb_cassert(mc, dl[0].mid < MDB_IDL_UM_MAX);
-		/* No - copy it */
 		np = mdb_page_malloc(txn, 1, 1);
 		if (!np) {
 			rc = ENOMEM;
@@ -2869,17 +2897,15 @@ mdb_page_touch(MDB_cursor *mc)
 		rc = mdb_mid2l_insert(dl, &mid);
 		mdb_cassert(mc, rc == 0);
 		np->mp_flags |= P_DIRTY;
-	} else {
-		return 0;
 	}
 
 	np_flags = np->mp_flags;	/* P_ADM_FLAGS */
 	mdb_page_copy(np, mp, txn->mt_env->me_psize);
 	np->mp_flags |= np_flags;
 	np->mp_pgno = pgno;
+	SET_PGTXNID(txn, np);
 
 done:
-	np->mp_txnid = txn->mt_txnid;
 	/* Adjust cursors pointing to mp */
 	mc->mc_pg[mc->mc_top] = np;
 	m2 = txn->mt_cursors[mc->mc_dbi];
@@ -3175,6 +3201,8 @@ mdb_txn_renew0(MDB_txn *txn)
 		txn->mt_child = NULL;
 		txn->mt_loose_pgs = NULL;
 		txn->mt_loose_count = 0;
+		txn->mt_workid = (txn->mt_txnid | MDB_PGTXNID_FLAGMASK) + 1;
+		txn->mt_last_workid = txn->mt_workid;
 		txn->mt_dirty_room = MDB_IDL_UM_MAX;
 		txn->mt_u.dirty_list = env->me_dirty_list;
 		txn->mt_u.dirty_list[0].mid = 0;
@@ -3239,6 +3267,33 @@ mdb_txn_renew(MDB_txn *txn)
 	return rc;
 }
 
+/** Used up all workids.  Rewind it and update dirty pages to match. */
+static txnid_t ESECT
+mdb_workid_rewind(MDB_txn *txn)
+{
+	txnid_t workid, diff;
+
+	while (txn->mt_parent)
+		txn = txn->mt_parent;
+	workid = txn->mt_txnid & ~MDB_PGTXNID_FLAGMASK;
+	do {
+		workid += MDB_PGTXNID_STEP;
+		diff = txn->mt_last_workid - workid;
+		if (diff) {
+			MDB_ID2L dl = txn->mt_u.dirty_list;
+			int i;
+			for (i = dl[0].mid; i; i--) {
+				if (MDB_PGTXNID_FLAGBITS)
+					((MDB_page *)dl[i].mptr)->mp_txnid -= diff;
+				else
+					((MDB_page *)dl[i].mptr)->mp_txnid = workid;
+			}
+			txn->mt_workid = txn->mt_last_workid = workid;
+		}
+	} while ((txn = txn->mt_child) != NULL);
+	return workid;
+}
+
 int
 mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 {
@@ -3294,6 +3349,10 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 
 	if (parent) {
 		unsigned int i;
+		txnid_t workid = parent->mt_last_workid + MDB_PGTXNID_STEP;
+		if (!workid) /* wraparound after lots of previous children */
+			workid = mdb_workid_rewind(parent) + MDB_PGTXNID_STEP;
+		txn->mt_workid = txn->mt_last_workid = workid;
 		txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs);
 		txn->mt_dbiseqs = parent->mt_dbiseqs;
 		txn->mt_u.dirty_list = malloc(sizeof(MDB_ID2)*MDB_IDL_UM_SIZE);
@@ -3786,7 +3845,7 @@ mdb_page_flush(MDB_txn *txn, int keep)
 	j = i = keep;
 
 	if (env->me_flags & MDB_WRITEMAP) {
-		/* Clear dirty flags */
+		/* Mark the pages as clean */
 		while (++i <= pagecount) {
 			dp = dl[i].mptr;
 			/* Don't flush this page yet */
@@ -3795,6 +3854,7 @@ mdb_page_flush(MDB_txn *txn, int keep)
 				dl[++j] = dl[i];
 				continue;
 			}
+			dp->mp_txnid = txn->mt_txnid;
 			dp->mp_flags &= ~P_DIRTY;
 		}
 		goto done;
@@ -3811,7 +3871,8 @@ mdb_page_flush(MDB_txn *txn, int keep)
 				continue;
 			}
 			pgno = dl[i].mid;
-			/* clear dirty flag */
+			/* Mark the page as clean */
+			dp->mp_txnid = txn->mt_txnid;
 			dp->mp_flags &= ~P_DIRTY;
 			pos = pgno * psize;
 			size = psize;
@@ -4128,6 +4189,7 @@ mdb_txn_commit(MDB_txn *txn)
 		*lp = txn->mt_loose_pgs;
 		parent->mt_loose_count += txn->mt_loose_count;
 
+		parent->mt_last_workid = txn->mt_last_workid;
 		parent->mt_child = NULL;
 		mdb_midl_free(((MDB_ntxn *)txn)->mnt_pgstate.mf_pghead);
 		free(txn);
@@ -8077,10 +8139,10 @@ current:
 		if (F_ISSET(leaf->mn_flags, F_BIGDATA)) {
 			MDB_page *omp;
 			MDB_ovpage ovp;
-			int level, ovpages, dpages = OVPAGES(data->mv_size, env->me_psize);
+			int ovpages, dpages = OVPAGES(data->mv_size, env->me_psize);
 
 			memcpy(&ovp, olddata.mv_data, sizeof(ovp));
-			if ((rc2 = MDB_PAGE_GET(mc, ovp.op_pgno, ovp.op_pages, &omp, &level)) != 0)
+			if ((rc2 = MDB_PAGE_GET(mc, ovp.op_pgno, ovp.op_pages, &omp, NULL)) != 0)
 				return rc2;
 			ovpages = ovp.op_pages;
 
@@ -8090,13 +8152,12 @@ current:
 				 * bother to try shrinking the page if the new data
 				 * is smaller than the overflow threshold.
 				 */
-				if (!(omp->mp_flags & P_DIRTY)) {
+				if (!IS_WRITABLE(mc->mc_txn, omp)) {
+				  if (!(omp->mp_flags & P_DIRTY)) {
 					rc = mdb_page_unspill(mc->mc_txn, omp, &omp);
 					if (rc)
 						return rc;
-					level = 0;		/* dirty in this txn */
-				}
-				if (level > 1) {
+				  } else {
 					/* It is writable only in a parent txn */
 					size_t sz = (size_t) env->me_psize * ovpages, off;
 					MDB_page *np = mdb_page_malloc(mc->mc_txn, ovpages, 1);
@@ -8123,7 +8184,9 @@ current:
 						sz = PAGEHDRSZ;
 					}
 					memcpy(np, omp, sz); /* Copy beginning of page */
+					SET_PGTXNID(mc->mc_txn, np);
 					omp = np;
+				  }
 				}
 				SETDSZ(leaf, data->mv_size);
 				if (F_ISSET(flags, MDB_RESERVE))

From adfa8f758adce1233e8a87bb52a0e41a7ab02c8b Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Tue, 25 Jul 2017 21:27:36 +0200
Subject: [PATCH 332/504] Drop P_DIRTY, and MDB_WRITEMAP dirty/spill lists

mt_workid = mt_txnid when WRITEMAP, so dirty pages == "spilled"
pages and mdb_page_flush() does nothing.
---
 libraries/liblmdb/mdb.c | 109 +++++++++++++++++++---------------------
 1 file changed, 53 insertions(+), 56 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 1d051e3ba0..4dd4f6e7a2 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -975,7 +975,10 @@ typedef struct MDB_page_header {
 	 *	Thus an #MDB_txn can write to pages with mp_txnid >= txn.mt_workid.
 	 *	A page with smaller mp_txnid is dirty in an ancestor txn or clean.
 	 *
-	 *	txn.mt_workid > txn.mt_txnid, to tell apart spilled and dirty pages.
+	 *	Non-#MDB_WRITEMAP sets txn.mt_workid > txn.mt_txnid, to tell apart
+	 *	spilled and dirty pages.  WRITEMAP sets mt_workid = mt_txnid, since
+	 *	it does not copy/spill pages.  Thus (page.mp_txnid == txn.mt_txnid)
+	 *	says "spilled page" without WRITEMAP, "dirty page" with WRITEMAP.
 	 *
 	 *	Finally, ((dirty page).mp_txnid & #MDB_PGTXNID_FLAGMASK) can be used
 	 *	for flags with non-WRITEMAP; it keeps low bits in workid = 0.
@@ -992,14 +995,13 @@ typedef struct MDB_page_header {
 #define	P_LEAF		 0x02		/**< leaf page */
 #define	P_OVERFLOW	 0x04		/**< overflow page */
 #define	P_META		 0x08		/**< meta page */
-#define	P_DIRTY		 0x10		/**< dirty page, also set for #P_SUBP pages */
 #define	P_LEAF2		 0x20		/**< for #MDB_DUPFIXED records */
 #define	P_SUBP		 0x40		/**< for #MDB_DUPSORT sub-pages */
 #define	P_DIRTY_OVF	 0x2000		/**< page has dirty overflow nodes */
 #define	P_LOOSE		 0x4000		/**< page was dirtied then freed, can be reused */
 #define	P_KEEP		 0x8000		/**< leave this page alone during spill */
 /** Persistent flags for page administration rather than page contents */
-#define	P_ADM_FLAGS	 (P_DIRTY)
+#define	P_ADM_FLAGS	 0 /* later... */
 /** @} */
 	uint16_t	mh_flags;		/**< @ref mdb_page */
 #define mp_lower	mp_pb.pb.pb_lower
@@ -1058,6 +1060,8 @@ typedef struct MDB_page {
 	/** Test if a page is a sub page */
 #define IS_SUBP(p)	 F_ISSET((p)->mp_flags, P_SUBP)
 
+	/** Test if (this non-sub page is dirty && env is non-#MDB_WRITEMAP) */
+#define IS_DIRTY_NW(txn, p)	((p)->mp_txnid > (txn)->mt_txnid)
 	/** Test if this non-sub page belongs to the current snapshot */
 #define IS_MUTABLE(txn, p)	((p)->mp_txnid >= (txn)->mt_txnid)
 	/** Test if this non-sub page is writable in this txn (not an ancestor) */
@@ -1331,7 +1335,7 @@ struct MDB_txn {
 
 	/** Written to mp_txnid of dirty pages, to be fixed by #mdb_page_flush().
 	 *
-	 *	Value >= 1 + (parent ? parent.last_workid : txnid).
+	 *	Value >= WRITEMAP ? txnid : 1 + (parent ? parent.last_workid : txnid).
 	 *	See #MDB_page.%mp_txnid.
 	 *
 	 *	An MDB_txn can write to a page when page.mp_txnid >= txn.mt_workid.
@@ -1358,10 +1362,11 @@ struct MDB_txn {
 	/** The sorted list of dirty pages we temporarily wrote to disk
 	 *	because the dirty list was full. page numbers in here are
 	 *	shifted left by 1, deleted slots have the LSB set.
+	 *	Unused with #MDB_WRITEMAP, which does not use a dirty list.
 	 */
 	MDB_IDL		mt_spill_pgs;
 	union {
-		/** For write txns: Modified pages. Sorted when not MDB_WRITEMAP. */
+		/** For write txns: Modified pages, sorted. Unused when MDB_WRITEMAP. */
 		MDB_ID2L	dirty_list;
 		/** For read txns: This thread/txn's reader table slot, or NULL. */
 		MDB_reader	*reader;
@@ -1434,6 +1439,7 @@ struct MDB_txn {
 	 *	Includes ancestor txns' dirty pages not hidden by other txns'
 	 *	dirty/spilled pages. Thus commit(nested txn) has room to merge
 	 *	dirty_list into mt_parent after freeing hidden mt_parent pages.
+	 *	When #MDB_WRITEMAP, it is nonzero but otherwise irrelevant.
 	 */
 	unsigned int	mt_dirty_room;
 };
@@ -1586,7 +1592,9 @@ struct MDB_env {
 	MDB_page	*me_dpages;		/**< list of malloc'd blocks for re-use */
 	/** IDL of pages that became unused in a write txn */
 	MDB_IDL		me_free_pgs;
-	/** ID2L of pages written during a write txn. Length MDB_IDL_UM_SIZE. */
+	/** ID2L of pages written during a write txn. Length MDB_IDL_UM_SIZE.
+	 *	Unused except for a dummy element when #MDB_WRITEMAP.
+	 */
 	MDB_ID2L	me_dirty_list;
 	/** Max number of freelist items that can fit in a single overflow page */
 	int			me_maxfree_1pg;
@@ -1919,7 +1927,7 @@ void
 mdb_page_list(MDB_page *mp)
 {
 	pgno_t pgno = mdb_dbg_pgno(mp);
-	const char *type, *state = (mp->mp_flags & P_DIRTY) ? ", dirty" : "";
+	const char *type;
 	MDB_node *node;
 	unsigned int i, nkeys, nsize, total = 0;
 	MDB_val key;
@@ -1932,8 +1940,7 @@ mdb_page_list(MDB_page *mp)
 	case P_LEAF|P_LEAF2:        type = "LEAF2 page";		break;
 	case P_LEAF|P_LEAF2|P_SUBP: type = "LEAF2 sub-page";	break;
 	case P_OVERFLOW:
-		fprintf(stderr, "Overflow page %"Yu" pages %u%s\n",
-			pgno, mp->mp_pages, state);
+		fprintf(stderr, "Overflow page %"Yu" pages %u\n", pgno, mp->mp_pages);
 		return;
 	case P_META:
 		fprintf(stderr, "Meta-page %"Yu" txnid %"Yu"\n",
@@ -1945,7 +1952,7 @@ mdb_page_list(MDB_page *mp)
 	}
 
 	nkeys = NUMKEYS(mp);
-	fprintf(stderr, "%s %"Yu" numkeys %d%s\n", type, pgno, nkeys, state);
+	fprintf(stderr, "%s %"Yu" numkeys %d\n", type, pgno, nkeys);
 
 	for (i=0; i<nkeys; i++) {
 		if (IS_LEAF2(mp)) {	/* LEAF2 pages have no mp_ptrs[] or node headers */
@@ -2188,7 +2195,7 @@ mdb_page_unref(MDB_txn *txn, MDB_page *mp)
 	pgno_t pgno;
 	MDB_ID3L tl = txn->mt_rpages;
 	unsigned x, rem;
-	if (mp->mp_flags & (P_SUBP|P_DIRTY))
+	if (IS_SUBP(mp) || IS_DIRTY_NW(txn, mp))
 		return;
 	rem = mp->mp_pgno & (MDB_RPAGE_CHUNK-1);
 	pgno = mp->mp_pgno ^ rem;
@@ -2264,14 +2271,14 @@ mdb_page_loose(MDB_cursor *mc, MDB_page *mp)
 /** Set or clear P_KEEP in dirty, non-overflow, non-sub pages watched by txn.
  * @param[in] mc A cursor handle for the current operation.
  * @param[in] pflags Flags of the pages to update:
- * P_DIRTY to set P_KEEP, P_DIRTY|P_KEEP to clear it.
+ * 0 to set P_KEEP, P_KEEP to clear it.
  * @param[in] all No shortcuts. Needed except after a full #mdb_page_flush().
  * @return 0 on success, non-zero on failure.
  */
 static int
 mdb_pages_xkeep(MDB_cursor *mc, unsigned pflags, int all)
 {
-	enum { Mask = P_SUBP|P_DIRTY|P_LOOSE|P_KEEP };
+	enum { Mask = P_SUBP|P_LOOSE|P_KEEP };
 	MDB_txn *txn = mc->mc_txn;
 	MDB_cursor *m3, *m0 = mc;
 	MDB_xcursor *mx;
@@ -2288,6 +2295,7 @@ mdb_pages_xkeep(MDB_cursor *mc, unsigned pflags, int all)
 				for (j=0; j<m3->mc_snum; j++) {
 					mp = m3->mc_pg[j];
 					if ((mp->mp_flags & Mask) == pflags)
+					  if (IS_DIRTY_NW(txn, mp))
 						mp->mp_flags ^= P_KEEP;
 				}
 				mx = m3->mc_xcursor;
@@ -2343,7 +2351,7 @@ static int mdb_page_flush(MDB_txn *txn, int keep);
  *
  * Otherwise, if not using nested txns, it is expected that apps will
  * not run into #MDB_TXN_FULL any more. The pages are flushed to disk
- * the same way as for a txn commit, e.g. their P_DIRTY flag is cleared.
+ * the same way as for a txn commit.
  * If the txn never references them again, they can be left alone.
  * If the txn only reads them, they can be used without any fuss.
  * If the txn writes them again, they can be dirtied immediately without
@@ -2372,7 +2380,7 @@ mdb_page_spill(MDB_cursor *m0, MDB_val *key, MDB_val *data)
 	unsigned int i, j, need;
 	int rc;
 
-	if (m0->mc_flags & C_SUB)
+	if (m0->mc_flags & (C_SUB|C_WRITEMAP))
 		return MDB_SUCCESS;
 
 	/* Estimate how much space this op will take */
@@ -2406,7 +2414,7 @@ mdb_page_spill(MDB_cursor *m0, MDB_val *key, MDB_val *data)
 	}
 
 	/* Preserve pages which may soon be dirtied again */
-	if ((rc = mdb_pages_xkeep(m0, P_DIRTY, 1)) != MDB_SUCCESS)
+	if ((rc = mdb_pages_xkeep(m0, 0, 1)) != MDB_SUCCESS)
 		goto done;
 
 	/* Less aggressive spill - we originally spilled the entire dirty list,
@@ -2454,7 +2462,7 @@ mdb_page_spill(MDB_cursor *m0, MDB_val *key, MDB_val *data)
 		goto done;
 
 	/* Reset any dirty pages we kept that page_flush didn't see */
-	rc = mdb_pages_xkeep(m0, P_DIRTY|P_KEEP, i);
+	rc = mdb_pages_xkeep(m0, P_KEEP, i);
 
 done:
 	txn->mt_flags |= rc ? MDB_TXN_ERROR : MDB_TXN_SPILLS;
@@ -2480,23 +2488,20 @@ mdb_find_oldest(MDB_txn *txn)
 	return oldest;
 }
 
-/** Mark a page as dirty and add it to the txn's dirty list */
+/** Add a page to the txn's dirty list, if there is one */
 static void
 mdb_page_dirty(MDB_txn *txn, MDB_page *mp)
 {
 	MDB_ID2 mid;
-	int rc, (*insert)(MDB_ID2L, MDB_ID2 *);
-
-	mp->mp_flags |= P_DIRTY;
+	int rc;
 
 	if (txn->mt_flags & MDB_TXN_WRITEMAP) {
-		insert = mdb_mid2l_append;
-	} else {
-		insert = mdb_mid2l_insert;
+		txn->mt_flags |= MDB_TXN_DIRTY;
+		return;
 	}
 	mid.mid = mp->mp_pgno;
 	mid.mptr = mp;
-	rc = insert(txn->mt_u.dirty_list, &mid);
+	rc = mdb_mid2l_insert(txn->mt_u.dirty_list, &mid);
 	mdb_tassert(txn, rc == 0);
 	txn->mt_dirty_room--;
 }
@@ -2806,9 +2811,7 @@ mdb_page_unspill(MDB_txn *txn, MDB_page *mp, MDB_page **ret)
 				num = mp->mp_pages;
 			else
 				num = 1;
-			if (env->me_flags & MDB_WRITEMAP) {
-				np = mp;
-			} else {
+			{
 				np = mdb_page_malloc(txn, num, 1);
 				if (!np)
 					return ENOMEM;
@@ -2873,7 +2876,7 @@ mdb_page_touch(MDB_cursor *mc)
 		} else {
 			mc->mc_db->md_root = pgno;
 		}
-	} else if (!F_ISSET(mp->mp_flags, P_DIRTY)) {
+	} else if (!IS_DIRTY_NW(txn, mp)) {
 		rc = mdb_page_unspill(txn, mp, &np);
 		if (rc)
 			goto fail;
@@ -2896,7 +2899,6 @@ mdb_page_touch(MDB_cursor *mc)
 		mid.mptr = np;
 		rc = mdb_mid2l_insert(dl, &mid);
 		mdb_cassert(mc, rc == 0);
-		np->mp_flags |= P_DIRTY;
 	}
 
 	np_flags = np->mp_flags;	/* P_ADM_FLAGS */
@@ -3201,9 +3203,14 @@ mdb_txn_renew0(MDB_txn *txn)
 		txn->mt_child = NULL;
 		txn->mt_loose_pgs = NULL;
 		txn->mt_loose_count = 0;
-		txn->mt_workid = (txn->mt_txnid | MDB_PGTXNID_FLAGMASK) + 1;
+		if (env->me_flags & MDB_WRITEMAP) {
+			txn->mt_workid = txn->mt_txnid;
+			txn->mt_dirty_room = 1;
+		} else {
+			txn->mt_workid = (txn->mt_txnid | MDB_PGTXNID_FLAGMASK) + 1;
+			txn->mt_dirty_room = MDB_IDL_UM_MAX;
+		}
 		txn->mt_last_workid = txn->mt_workid;
-		txn->mt_dirty_room = MDB_IDL_UM_MAX;
 		txn->mt_u.dirty_list = env->me_dirty_list;
 		txn->mt_u.dirty_list[0].mid = 0;
 		txn->mt_free_pgs = env->me_free_pgs;
@@ -3845,18 +3852,6 @@ mdb_page_flush(MDB_txn *txn, int keep)
 	j = i = keep;
 
 	if (env->me_flags & MDB_WRITEMAP) {
-		/* Mark the pages as clean */
-		while (++i <= pagecount) {
-			dp = dl[i].mptr;
-			/* Don't flush this page yet */
-			if (dp->mp_flags & (P_LOOSE|P_KEEP)) {
-				dp->mp_flags &= ~P_KEEP;
-				dl[++j] = dl[i];
-				continue;
-			}
-			dp->mp_txnid = txn->mt_txnid;
-			dp->mp_flags &= ~P_DIRTY;
-		}
 		goto done;
 	}
 
@@ -3873,7 +3868,6 @@ mdb_page_flush(MDB_txn *txn, int keep)
 			pgno = dl[i].mid;
 			/* Mark the page as clean */
 			dp->mp_txnid = txn->mt_txnid;
-			dp->mp_flags &= ~P_DIRTY;
 			pos = pgno * psize;
 			size = psize;
 #if MDB_RPAGE_CACHE
@@ -4023,11 +4017,11 @@ retry_seek:
 		}
 		mdb_dpage_free(env, dp);
 	}
-
-done:
 	i--;
 	txn->mt_dirty_room += i - j;
 	dl[0].mid = j;
+
+done:
 	return MDB_SUCCESS;
 }
 
@@ -5765,8 +5759,10 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 		/* silently ignore WRITEMAP when we're only getting read access */
 		flags &= ~MDB_WRITEMAP;
 	} else {
+		/* WRITEMAP has a dummy element to match dirty_room = 1 */
+		size_t dl_size = (flags & MDB_WRITEMAP) ? 2 : MDB_IDL_UM_SIZE;
 		if (!((env->me_free_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX)) &&
-			  (env->me_dirty_list = calloc(MDB_IDL_UM_SIZE, sizeof(MDB_ID2)))))
+			  (env->me_dirty_list = calloc(dl_size, sizeof(MDB_ID2)))))
 			rc = ENOMEM;
 	}
 
@@ -6956,18 +6952,19 @@ mdb_ovpage_free(MDB_cursor *mc, MDB_page *mp)
 		rc = mdb_midl_need(&env->me_pghead, ovpages);
 		if (rc)
 			return rc;
-		if (!(mp->mp_flags & P_DIRTY)) {
-			MDB_IDL sl = txn->mt_spill_pgs;
-			if (sl)
-				x = mdb_midl_search(sl, pn);
-			if (! (sl && x <= sl[0] && sl[x] == pn))
+		if (!IS_DIRTY_NW(txn, mp)) { /* spilled or WRITEMAP */
+		  MDB_IDL sl = txn->mt_spill_pgs;
+		  if (sl) {
+			x = mdb_midl_search(sl, pn);
+			if (! (x <= sl[0] && sl[x] == pn))
 				return MDB_PROBLEM;
 			/* This page is no longer spilled */
 			if (x == sl[0])
 				sl[0]--;
 			else
 				sl[x] |= 1;
-			goto release;
+		  }
+		  goto release;
 		}
 		/* Remove from dirty list */
 		dl = txn->mt_u.dirty_list;
@@ -8037,7 +8034,7 @@ more:
 				dkey.mv_data = memcpy(fp+1, olddata.mv_data, olddata.mv_size);
 
 				/* Make sub-page header for the dup items, with dummy body */
-				fp->mp_flags = P_LEAF|P_DIRTY|P_SUBP;
+				fp->mp_flags = P_LEAF|P_SUBP;
 				fp->mp_lower = (PAGEHDRSZ-PAGEBASE);
 				xdata.mv_size = PAGEHDRSZ + dkey.mv_size + data->mv_size;
 				if (mc->mc_db->md_flags & MDB_DUPFIXED) {
@@ -8153,7 +8150,7 @@ current:
 				 * is smaller than the overflow threshold.
 				 */
 				if (!IS_WRITABLE(mc->mc_txn, omp)) {
-				  if (!(omp->mp_flags & P_DIRTY)) {
+				  if (!IS_DIRTY_NW(mc->mc_txn, omp)) {
 					rc = mdb_page_unspill(mc->mc_txn, omp, &omp);
 					if (rc)
 						return rc;

From 81984ab22865d9dcd862e8de25a79f81530dc300 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 26 Jul 2017 20:52:31 +0100
Subject: [PATCH 333/504] Fix 418ea8b94e SEGV on spilled ovpg

Don't spill an ovpg that is referenced in a cursor
---
 libraries/liblmdb/mdb.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 4dd4f6e7a2..95d24803a4 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2298,6 +2298,9 @@ mdb_pages_xkeep(MDB_cursor *mc, unsigned pflags, int all)
 					  if (IS_DIRTY_NW(txn, mp))
 						mp->mp_flags ^= P_KEEP;
 				}
+				if (MC_OVPG(m3) && ((MC_OVPG(m3)->mp_flags & Mask) == pflags) &&
+					IS_DIRTY_NW(txn, MC_OVPG(m3)))
+					MC_OVPG(m3)->mp_flags ^= P_KEEP;
 				mx = m3->mc_xcursor;
 				/* Proceed to mx if it is at a sub-database */
 				if (! (mx && (mx->mx_cursor.mc_flags & C_INITIALIZED)))

From 499866aba7cb9e3e3c4209a67da524d6adf8b8e2 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 26 Jul 2017 21:37:40 +0100
Subject: [PATCH 334/504] ITS#8699 more for cursor_del ITS#8622

Set C_DEL flag on reinit'd subcursor
---
 libraries/liblmdb/mdb.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 95d24803a4..149899a3b8 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -9799,8 +9799,10 @@ mdb_cursor_del0(MDB_cursor *mc)
 							if (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) {
 								if (!(node->mn_flags & F_SUBDATA))
 									m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node);
-							} else
+							} else {
 								mdb_xcursor_init1(m3, node);
+								m3->mc_xcursor->mx_cursor.mc_flags |= C_DEL;
+							}
 						}
 					}
 				}

From 972917449de3838f22cf3dc5456dc8ded3d3355f Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Thu, 27 Jul 2017 08:19:03 +0200
Subject: [PATCH 335/504] Keep MC_OVPG() == NULL when not MDB_REMAPPING()

Needed since 418ea8b94e14567ba2be9f9772f38f563a0d7e9c introduced
option MDB_REMAP_CHUNKS.
---
 libraries/liblmdb/mdb.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 149899a3b8..7c0343447d 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1502,7 +1502,9 @@ struct MDB_cursor {
 #if MDB_RPAGE_CACHE
 	MDB_page	*mc_ovpg;		/**< a referenced overflow page */
 #	define MC_OVPG(mc)			((mc)->mc_ovpg)
-#	define MC_SET_OVPG(mc, pg)	((mc)->mc_ovpg = (pg))
+#	define MC_SET_OVPG(mc, pg)	\
+		((mc)->mc_ovpg = \
+		 (MDB_REMAPPING((mc)->mc_txn->mt_env->me_flags) ? (pg) : NULL))
 #else
 #	define MC_OVPG(mc)			((MDB_page *)0)
 #	define MC_SET_OVPG(mc, pg)	((void)0)

From 24f437df86d73d3c440928231fbec616db605c79 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Thu, 27 Jul 2017 08:22:34 +0200
Subject: [PATCH 336/504] Less hungry mdb_pages_xkeep()

Skip pages that are dirty only in an ancestor txn
---
 libraries/liblmdb/mdb.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 7c0343447d..92b9fcde38 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2297,11 +2297,11 @@ mdb_pages_xkeep(MDB_cursor *mc, unsigned pflags, int all)
 				for (j=0; j<m3->mc_snum; j++) {
 					mp = m3->mc_pg[j];
 					if ((mp->mp_flags & Mask) == pflags)
-					  if (IS_DIRTY_NW(txn, mp))
+					  if (IS_WRITABLE(txn, mp))
 						mp->mp_flags ^= P_KEEP;
 				}
 				if (MC_OVPG(m3) && ((MC_OVPG(m3)->mp_flags & Mask) == pflags) &&
-					IS_DIRTY_NW(txn, MC_OVPG(m3)))
+					IS_WRITABLE(txn, MC_OVPG(m3)))
 					MC_OVPG(m3)->mp_flags ^= P_KEEP;
 				mx = m3->mc_xcursor;
 				/* Proceed to mx if it is at a sub-database */

From 61b5b6270df4317012bb77f013dc8a40c960e359 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 29 Jul 2017 07:56:46 +0200
Subject: [PATCH 337/504] Doc fixes

---
 libraries/liblmdb/lmdb.h | 4 ++--
 libraries/liblmdb/mdb.c  | 6 ++++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index c471706d8b..d691264657 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -995,8 +995,8 @@ int  mdb_env_set_assert(MDB_env *env, MDB_assert_func *func);
 #if MDB_RPAGE_CACHE
 	/** @brief Set encryption on an environment.
 	 *
-	 * This must be called before #mdb_env_open(). It implicitly sets #MDB_REMAP_CHUNK
-	 * on the env.
+	 * This must be called before #mdb_env_open().
+	 * It implicitly sets #MDB_REMAP_CHUNKS on the env.
 	 * @param[in] env An environment handle returned by #mdb_env_create().
 	 * @param[in] func An #MDB_enc_func function.
 	 * @param[in] key An array of two values: key[0] is the encryption key,
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 92b9fcde38..a8803ca97a 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -471,7 +471,7 @@ typedef pthread_mutex_t *mdb_mutexref_t;
 #define MNAME_LEN	(sizeof(pthread_mutex_t))
 #endif
 
-/** Initial part of #MDB_env.me_mutexname[].
+/** Initial part of #MDB_env.%me_mutexname[].
  *	Changes to this code must be reflected in #MDB_LOCK_FORMAT.
  */
 #ifdef _WIN32
@@ -6322,7 +6322,8 @@ static void mdb_rpage_decrypt(MDB_env *env, MDB_ID3 *id3, int rem, int numpgs);
  * reference to one overflow page at a time.
  *
  * @param[in] txn the transaction for this access.
- * @param[in] pgno the page number for the page to retrieve.
+ * @param[in] pg0 the page number for the page to retrieve.
+ * @param[in] numpgs number of database pages (can be > 1 for overflow pages)
  * @param[out] ret address of a pointer where the page's address will be stored.
  * @return 0 on success, non-zero on failure.
  */
@@ -6652,6 +6653,7 @@ static void mdb_rpage_dispose(MDB_env *env, MDB_ID3 *id3)
  * Set #MDB_TXN_ERROR on failure.
  * @param[in] mc the cursor accessing the page.
  * @param[in] pgno the page number for the page to retrieve.
+ * @param[in] numpgs number of database pages (can be > 1 for overflow pages)
  * @param[out] ret address of a pointer where the page's address will be stored.
  * @param[out] lvl dirty_list inheritance level of found page. 1=current txn, 0=mapped page.
  * @return 0 on success, non-zero on failure.

From d6cf4761692242391fdc9617d54e4063e490dbd4 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 29 Jul 2017 07:59:01 +0200
Subject: [PATCH 338/504] Drop mdb_page_get() arg 'level', it's always NULL

---
 libraries/liblmdb/mdb.c | 46 +++++++++++++++++------------------------
 1 file changed, 19 insertions(+), 27 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index a8803ca97a..3fd47ddcb4 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1675,13 +1675,13 @@ enum {
 static void mdb_txn_end(MDB_txn *txn, unsigned mode);
 
 #if MDB_RPAGE_CACHE
-static int  mdb_page_get(MDB_cursor *mc, pgno_t pgno, int numpgs, MDB_page **mp, int *lvl);
-#define MDB_PAGE_GET(mc, pg, numpgs, mp, lvl)	mdb_page_get(mc, pg, numpgs, mp, lvl)
+#define MDB_PAGE_GET(mc, pg, numpgs, mp)	mdb_page_get(mc, pg, numpgs, mp)
 static void mdb_rpage_dispose(MDB_env *env, MDB_ID3 *id3);
 #else
-static int  mdb_page_get(MDB_cursor *mc, pgno_t pgno, MDB_page **mp, int *lvl);
-#define MDB_PAGE_GET(mc, pg, numpgs, mp, lvl)	mdb_page_get(mc, pg, mp, lvl)
+/* Drop unused numpgs argument when !MDB_RPAGE_CACHE */
+#define MDB_PAGE_GET(mc, pg, numpgs, mp)	mdb_page_get(mc, pg, mp)
 #endif
+static int  MDB_PAGE_GET(MDB_cursor *mc, pgno_t pgno, int numpgs, MDB_page **mp);
 
 static int  mdb_page_search_root(MDB_cursor *mc,
 			    MDB_val *key, int modify);
@@ -6655,7 +6655,6 @@ static void mdb_rpage_dispose(MDB_env *env, MDB_ID3 *id3)
  * @param[in] pgno the page number for the page to retrieve.
  * @param[in] numpgs number of database pages (can be > 1 for overflow pages)
  * @param[out] ret address of a pointer where the page's address will be stored.
- * @param[out] lvl dirty_list inheritance level of found page. 1=current txn, 0=mapped page.
  * @return 0 on success, non-zero on failure.
  */
 static int
@@ -6663,15 +6662,13 @@ mdb_page_get(MDB_cursor *mc, pgno_t pgno,
 #if MDB_RPAGE_CACHE
 	int numpgs,
 #endif
-	MDB_page **ret, int *lvl)
+	MDB_page **ret)
 {
 	MDB_txn *txn = mc->mc_txn;
 	MDB_page *p = NULL;
-	int level;
 
 	if (! (mc->mc_flags & (C_ORIG_RDONLY|C_WRITEMAP))) {
 		MDB_txn *tx2 = txn;
-		level = 1;
 		do {
 			MDB_ID2L dl = tx2->mt_u.dirty_list;
 			unsigned x;
@@ -6694,7 +6691,6 @@ mdb_page_get(MDB_cursor *mc, pgno_t pgno,
 					goto done;
 				}
 			}
-			level++;
 		} while ((tx2 = tx2->mt_parent) != NULL);
 	}
 
@@ -6704,8 +6700,6 @@ mdb_page_get(MDB_cursor *mc, pgno_t pgno,
 		return MDB_PAGE_NOTFOUND;
 	}
 
-	level = 0;
-
 mapped:
 #if MDB_RPAGE_CACHE
 	if (MDB_REMAPPING(txn->mt_env->me_flags)) {
@@ -6723,8 +6717,6 @@ mapped:
 
 done:
 	*ret = p;
-	if (lvl)
-		*lvl = level;
 	return MDB_SUCCESS;
 }
 
@@ -6781,7 +6773,7 @@ mdb_page_search_root(MDB_cursor *mc, MDB_val *key, int flags)
 		mdb_cassert(mc, i < NUMKEYS(mp));
 		node = NODEPTR(mp, i);
 
-		if ((rc = MDB_PAGE_GET(mc, NODEPGNO(node), 1, &mp, NULL)) != 0)
+		if ((rc = MDB_PAGE_GET(mc, NODEPGNO(node), 1, &mp)) != 0)
 			return rc;
 
 		mc->mc_ki[mc->mc_top] = i;
@@ -6824,7 +6816,7 @@ mdb_page_search_lowest(MDB_cursor *mc)
 	MDB_node	*node = NODEPTR(mp, 0);
 	int rc;
 
-	if ((rc = MDB_PAGE_GET(mc, NODEPGNO(node), 1, &mp, NULL)) != 0)
+	if ((rc = MDB_PAGE_GET(mc, NODEPGNO(node), 1, &mp)) != 0)
 		return rc;
 
 	mc->mc_ki[mc->mc_top] = 0;
@@ -6904,7 +6896,7 @@ mdb_page_search(MDB_cursor *mc, MDB_val *key, int flags)
 		if (mc->mc_pg[0])
 			MDB_PAGE_UNREF(mc->mc_txn, mc->mc_pg[0]);
 #endif
-		if ((rc = MDB_PAGE_GET(mc, root, 1, &mc->mc_pg[0], NULL)) != 0)
+		if ((rc = MDB_PAGE_GET(mc, root, 1, &mc->mc_pg[0])) != 0)
 			return rc;
 	}
 
@@ -7037,7 +7029,7 @@ mdb_node_read(MDB_cursor *mc, MDB_node *leaf, MDB_val *data)
 	 */
 	memcpy(&ovp, NODEDATA(leaf), sizeof(ovp));
 	{
-	if ((rc = MDB_PAGE_GET(mc, ovp.op_pgno, ovp.op_pages, &omp, NULL)) != 0) {
+	if ((rc = MDB_PAGE_GET(mc, ovp.op_pgno, ovp.op_pages, &omp)) != 0) {
 		DPRINTF(("read overflow page %"Yu" failed", ovp.op_pgno));
 		return rc;
 	}
@@ -7126,7 +7118,7 @@ mdb_cursor_sibling(MDB_cursor *mc, int move_right)
 	MDB_PAGE_UNREF(mc->mc_txn, op);
 
 	indx = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
-	if ((rc = MDB_PAGE_GET(mc, NODEPGNO(indx), 1, &mp, NULL)) != 0) {
+	if ((rc = MDB_PAGE_GET(mc, NODEPGNO(indx), 1, &mp)) != 0) {
 		/* mc will be inconsistent if caller does mc_snum++ as above */
 		mc->mc_flags &= ~(C_INITIALIZED|C_EOF);
 		return rc;
@@ -8146,7 +8138,7 @@ current:
 			int ovpages, dpages = OVPAGES(data->mv_size, env->me_psize);
 
 			memcpy(&ovp, olddata.mv_data, sizeof(ovp));
-			if ((rc2 = MDB_PAGE_GET(mc, ovp.op_pgno, ovp.op_pages, &omp, NULL)) != 0)
+			if ((rc2 = MDB_PAGE_GET(mc, ovp.op_pgno, ovp.op_pages, &omp)) != 0)
 				return rc2;
 			ovpages = ovp.op_pages;
 
@@ -8442,7 +8434,7 @@ mdb_cursor_del(MDB_cursor *mc, unsigned int flags)
 		MDB_ovpage ovp;
 
 		memcpy(&ovp, NODEDATA(leaf), sizeof(ovp));
-		if ((rc = MDB_PAGE_GET(mc, ovp.op_pgno, ovp.op_pages, &omp, NULL)) ||
+		if ((rc = MDB_PAGE_GET(mc, ovp.op_pgno, ovp.op_pages, &omp)) ||
 			(rc = mdb_ovpage_free(mc, omp)))
 			goto fail;
 	}
@@ -9609,7 +9601,7 @@ mdb_rebalance(MDB_cursor *mc)
 			if (rc)
 				return rc;
 			mc->mc_db->md_root = NODEPGNO(NODEPTR(mp, 0));
-			rc = MDB_PAGE_GET(mc, mc->mc_db->md_root, 1, &mc->mc_pg[0], NULL);
+			rc = MDB_PAGE_GET(mc, mc->mc_db->md_root, 1, &mc->mc_pg[0]);
 			if (rc)
 				return rc;
 			mc->mc_db->md_depth--;
@@ -9670,7 +9662,7 @@ mdb_rebalance(MDB_cursor *mc)
 		DPUTS("reading right neighbor");
 		mn.mc_ki[ptop]++;
 		node = NODEPTR(mc->mc_pg[ptop], mn.mc_ki[ptop]);
-		rc = MDB_PAGE_GET(mc, NODEPGNO(node), 1, &mn.mc_pg[mn.mc_top], NULL);
+		rc = MDB_PAGE_GET(mc, NODEPGNO(node), 1, &mn.mc_pg[mn.mc_top]);
 		if (rc)
 			return rc;
 		mn.mc_ki[mn.mc_top] = 0;
@@ -9682,7 +9674,7 @@ mdb_rebalance(MDB_cursor *mc)
 		DPUTS("reading left neighbor");
 		mn.mc_ki[ptop]--;
 		node = NODEPTR(mc->mc_pg[ptop], mn.mc_ki[ptop]);
-		rc = MDB_PAGE_GET(mc, NODEPGNO(node), 1, &mn.mc_pg[mn.mc_top], NULL);
+		rc = MDB_PAGE_GET(mc, NODEPGNO(node), 1, &mn.mc_pg[mn.mc_top]);
 		if (rc)
 			return rc;
 		mn.mc_ki[mn.mc_top] = NUMKEYS(mn.mc_pg[mn.mc_top]) - 1;
@@ -10479,7 +10471,7 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags)
 	mc.mc_txn = my->mc_txn;
 	mc.mc_flags = my->mc_txn->mt_flags & (C_ORIG_RDONLY|C_WRITEMAP);
 
-	rc = MDB_PAGE_GET(&mc, *pg, 1, &mc.mc_pg[0], NULL);
+	rc = MDB_PAGE_GET(&mc, *pg, 1, &mc.mc_pg[0]);
 	if (rc)
 		return rc;
 	rc = mdb_page_search_root(&mc, NULL, MDB_PS_FIRST);
@@ -10523,7 +10515,7 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags)
 						}
 
 						memcpy(&ovp, NODEDATA(ni), sizeof(ovp));
-						rc = MDB_PAGE_GET(&mc, ovp.op_pgno, ovp.op_pages, &omp, NULL);
+						rc = MDB_PAGE_GET(&mc, ovp.op_pgno, ovp.op_pages, &omp);
 						if (rc)
 							goto done;
 						if (my->mc_wlen[toggle] >= MDB_WBUF) {
@@ -10575,7 +10567,7 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags)
 again:
 				ni = NODEPTR(mp, mc.mc_ki[mc.mc_top]);
 				pg = NODEPGNO(ni);
-				rc = MDB_PAGE_GET(&mc, pg, 1, &mp, NULL);
+				rc = MDB_PAGE_GET(&mc, pg, 1, &mp);
 				if (rc)
 					goto done;
 				mc.mc_top++;
@@ -11249,7 +11241,7 @@ mdb_drop0(MDB_cursor *mc, int subs)
 		if (MDB_REMAPPING(mc->mc_txn->mt_env->me_flags)) {
 		/* bump refcount for mx's pages */
 		for (i=0; i<mc->mc_snum; i++)
-			MDB_PAGE_GET(&mx, mc->mc_pg[i]->mp_pgno, 1, &mx.mc_pg[i], NULL);
+			MDB_PAGE_GET(&mx, mc->mc_pg[i]->mp_pgno, 1, &mx.mc_pg[i]);
 		}
 
 		while (mc->mc_snum > 0) {

From 8b31a4a62b9187e2a54dc76ce26b6069383602fb Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 29 Jul 2017 07:59:39 +0200
Subject: [PATCH 339/504] mdb_page_get() can ignore the toplevel spill list

...when we search dirty list before instead of after spill list.
---
 libraries/liblmdb/mdb.c | 37 ++++++++++++++++++++++---------------
 1 file changed, 22 insertions(+), 15 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 3fd47ddcb4..cd150cf89d 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -6664,26 +6664,17 @@ mdb_page_get(MDB_cursor *mc, pgno_t pgno,
 #endif
 	MDB_page **ret)
 {
-	MDB_txn *txn = mc->mc_txn;
+	MDB_txn *txn = mc->mc_txn, *tx2;
 	MDB_page *p = NULL;
 
 	if (! (mc->mc_flags & (C_ORIG_RDONLY|C_WRITEMAP))) {
-		MDB_txn *tx2 = txn;
-		do {
+		for (tx2 = txn;; ) {
 			MDB_ID2L dl = tx2->mt_u.dirty_list;
+			MDB_IDL  sl;
 			unsigned x;
-			/* Spilled pages were dirtied in this txn and flushed
-			 * because the dirty list got full. Bring this page
-			 * back in from the map (but don't unspill it here,
-			 * leave that unless page_touch happens again).
+			/* tx2 may have malloced its own "dirty" version of the
+			 * page, with the same page number.
 			 */
-			if (tx2->mt_spill_pgs) {
-				MDB_ID pn = pgno << 1;
-				x = mdb_midl_search(tx2->mt_spill_pgs, pn);
-				if (x <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[x] == pn) {
-					goto mapped;
-				}
-			}
 			if (dl[0].mid) {
 				unsigned x = mdb_mid2l_search(dl, pgno);
 				if (x <= dl[0].mid && dl[x].mid == pgno) {
@@ -6691,7 +6682,23 @@ mdb_page_get(MDB_cursor *mc, pgno_t pgno,
 					goto done;
 				}
 			}
-		} while ((tx2 = tx2->mt_parent) != NULL);
+			/* Spilled pages were dirtied in this txn, then cleaned
+			 * and flushed to the map when dirty_list got full.
+			 * Check if tx2 spilled the page before moving on to
+			 * search the parent.  (But don't unspill here, leave
+			 * that unless page_touch happens again.)
+			 */
+			sl = tx2->mt_spill_pgs;
+			if ((tx2 = tx2->mt_parent) == NULL)
+				break;
+			if (sl) {
+				MDB_ID pn = pgno << 1;
+				x = mdb_midl_search(sl, pn);
+				if (x <= sl[0] && sl[x] == pn) {
+					goto mapped;
+				}
+			}
+		}
 	}
 
 	if (pgno >= txn->mt_next_pgno) {

From 6e1f74856d1e6ab42bf8463939708ef49a798d7c Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 29 Jul 2017 14:54:55 +0200
Subject: [PATCH 340/504] Rename mdb_env_getflags->envflags, clarify options

The old name resembled existing function mdb_env_get_flags().
---
 libraries/liblmdb/mdb.c | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index cd150cf89d..9d20799be2 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5658,14 +5658,15 @@ mdb_enctest(const MDB_val *src, MDB_val *dst, const MDB_val *key, int encdec)
  *	Supports the normal flags plus 'e' = trivial encryption for testing.
  */
 static int ESECT
-mdb_env_getflags(MDB_env *env)
+mdb_env_envflags(MDB_env *env)
 {
-	static const char names[] = "acfhilmnrstwe";
+	static const char names[] = "ace" "fhi" "lmn" "rst" "w";
 	static const unsigned f[] = {
-		MDB_MAPASYNC, MDB_REMAP_CHUNKS, MDB_FIXEDMAP, MDB_NORDAHEAD,
-		MDB_NOMEMINIT, MDB_NOLOCK, MDB_NOMETASYNC, MDB_NOSUBDIR,
-		MDB_RDONLY, MDB_NOSYNC, MDB_NOTLS, MDB_WRITEMAP,
-		MDB_ENCRYPT,
+		/*a*/ MDB_MAPASYNC, /*c*/ MDB_REMAP_CHUNKS, /*e*/ MDB_ENCRYPT,
+		/*f*/ MDB_FIXEDMAP, /*h*/ MDB_NORDAHEAD,    /*i*/ MDB_NOMEMINIT,
+		/*l*/ MDB_NOLOCK,   /*m*/ MDB_NOMETASYNC,   /*n*/ MDB_NOSUBDIR,
+		/*r*/ MDB_RDONLY,   /*s*/ MDB_NOSYNC,       /*t*/ MDB_NOTLS,
+		/*w*/ MDB_WRITEMAP,
 	};
 	unsigned flags = 0;
 	const char *s, *opts = getenv("LMDB_FLAGS");
@@ -5696,7 +5697,7 @@ mdb_env_getflags(MDB_env *env)
 	return MDB_SUCCESS;
 }
 #else
-#define mdb_env_getflags(env) MDB_SUCCESS
+#define mdb_env_envflags(env) MDB_SUCCESS
 #endif	/* MDB_TEST */
 
 	/** Only a subset of the @ref mdb_env flags can be changed
@@ -5720,7 +5721,7 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 	if (env->me_fd!=INVALID_HANDLE_VALUE || (flags & ~(CHANGEABLE|CHANGELESS)))
 		return EINVAL;
 
-	if ((rc = mdb_env_getflags(env)) != MDB_SUCCESS)
+	if ((rc = mdb_env_envflags(env)) != MDB_SUCCESS)
 		return rc;
 	flags |= env->me_flags;
 

From b4ddec0bb453579ebbaf2059f05ad639122b5882 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sat, 29 Jul 2017 16:31:06 +0200
Subject: [PATCH 341/504] Expose MDB_ENCRYPT flag

---
 libraries/liblmdb/mdb.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 9d20799be2..0911bbb6d6 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5707,8 +5707,9 @@ mdb_env_envflags(MDB_env *env)
 #define	CHANGEABLE	(MDB_NOSYNC|MDB_NOMETASYNC|MDB_MAPASYNC|MDB_NOMEMINIT)
 #define	CHANGELESS	(MDB_FIXEDMAP|MDB_NOSUBDIR|MDB_RDONLY| \
 	MDB_WRITEMAP|MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD|MDB_REMAP_CHUNKS)
+#define EXPOSED		(CHANGEABLE|CHANGELESS | MDB_ENCRYPT)
 
-#if VALID_FLAGS & PERSISTENT_FLAGS & (CHANGEABLE|CHANGELESS)
+#if VALID_FLAGS & PERSISTENT_FLAGS & EXPOSED
 # error "Persistent DB flags & env flags overlap, but both go in mm_flags"
 #endif
 
@@ -10909,7 +10910,7 @@ mdb_env_get_flags(MDB_env *env, unsigned int *arg)
 	if (!env || !arg)
 		return EINVAL;
 
-	*arg = env->me_flags & (CHANGEABLE|CHANGELESS);
+	*arg = env->me_flags & EXPOSED;
 	return MDB_SUCCESS;
 }
 

From 0158f67c14e4be93fe8ad5e6ff22f020de65f7b8 Mon Sep 17 00:00:00 2001
From: moneromooo-monero <moneromooo-monero@users.noreply.github.com>
Date: Mon, 31 Jul 2017 00:04:28 +0100
Subject: [PATCH 342/504] ITS#8704 add MDB_PREVMETA flag to mdb_env_open

used to open the previous meta page, in case the latest one
is corrupted

From https://github.com/LMDB/lmdb/pull/12
---
 libraries/liblmdb/lmdb.h |  8 +++++++-
 libraries/liblmdb/mdb.c  | 15 ++++++++-------
 2 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index d691264657..3a6be034a4 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -354,8 +354,10 @@ typedef void (MDB_enc_func)(const MDB_val *src, MDB_val *dst, const MDB_val *key
 #define MDB_NORDAHEAD	0x800000
 	/** don't initialize malloc'd memory before writing to datafile */
 #define MDB_NOMEMINIT	0x1000000
+	/** use the previous meta page rather than the latest one */
+#define MDB_PREVMETA	0x2000000
 	/** don't use a single mmap, remap individual chunks (needs MDB_RPAGE_CACHE) */
-#define MDB_REMAP_CHUNKS	0x2000000
+#define MDB_REMAP_CHUNKS	0x4000000
 /** @} */
 
 /**	@defgroup	mdb_dbi_open	Database Flags
@@ -670,6 +672,10 @@ int  mdb_env_create(MDB_env **env);
 	 *		caller is expected to overwrite all of the memory that was
 	 *		reserved in that case.
 	 *		This flag may be changed at any time using #mdb_env_set_flags().
+	 *	<li>#MDB_PREVMETA
+	 *		Open the environment with the previous meta page rather than the latest
+	 *		one. This loses the latest transaction, but may help work around some
+	 *		types of corruption.
 	 * </ul>
 	 * @param[in] mode The UNIX permissions to set on created files and semaphores.
 	 * This parameter is ignored on Windows.
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 0911bbb6d6..e14b8656af 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1697,7 +1697,7 @@ static int	mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst);
 static int	mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata,
 				pgno_t newpgno, unsigned int nflags);
 
-static int  mdb_env_read_header(MDB_env *env, MDB_meta *meta);
+static int  mdb_env_read_header(MDB_env *env, int prev, MDB_meta *meta);
 static MDB_meta *mdb_env_pick_meta(const MDB_env *env);
 static int  mdb_env_write_meta(MDB_txn *txn);
 #ifdef MDB_USE_POSIX_MUTEX /* Drop unused excl arg */
@@ -4270,11 +4270,12 @@ fail:
 /** Read the environment parameters of a DB environment before
  * mapping it into memory.
  * @param[in] env the environment handle
+ * @param[in] prev whether to read the backup meta page
  * @param[out] meta address of where to store the meta information
  * @return 0 on success, non-zero on failure.
  */
 static int ESECT
-mdb_env_read_header(MDB_env *env, MDB_meta *meta)
+mdb_env_read_header(MDB_env *env, int prev, MDB_meta *meta)
 {
 	MDB_metabuf	pbuf;
 	MDB_page	*p;
@@ -4325,7 +4326,7 @@ mdb_env_read_header(MDB_env *env, MDB_meta *meta)
 			return MDB_VERSION_MISMATCH;
 		}
 
-		if (off == 0 || m->mm_txnid > meta->mm_txnid)
+		if (off == 0 || (prev ? m->mm_txnid < meta->mm_txnid : m->mm_txnid > meta->mm_txnid))
 			*meta = *m;
 	}
 	return 0;
@@ -4965,7 +4966,7 @@ mdb_fopen(const MDB_env *env, MDB_name *fname,
 /** Further setup required for opening an LMDB environment
  */
 static int ESECT
-mdb_env_open2(MDB_env *env)
+mdb_env_open2(MDB_env *env, int prev)
 {
 	unsigned int flags = env->me_flags;
 	int i, newenv = 0, rc;
@@ -5028,7 +5029,7 @@ mdb_env_open2(MDB_env *env)
 	}
 #endif
 
-	if ((i = mdb_env_read_header(env, &meta)) != 0) {
+	if ((i = mdb_env_read_header(env, prev, &meta)) != 0) {
 		if (i != ENOENT)
 			return i;
 		DPUTS("new mdbenv");
@@ -5706,7 +5707,7 @@ mdb_env_envflags(MDB_env *env)
 	 */
 #define	CHANGEABLE	(MDB_NOSYNC|MDB_NOMETASYNC|MDB_MAPASYNC|MDB_NOMEMINIT)
 #define	CHANGELESS	(MDB_FIXEDMAP|MDB_NOSUBDIR|MDB_RDONLY| \
-	MDB_WRITEMAP|MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD|MDB_REMAP_CHUNKS)
+	MDB_WRITEMAP|MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD|MDB_PREVMETA|MDB_REMAP_CHUNKS)
 #define EXPOSED		(CHANGEABLE|CHANGELESS | MDB_ENCRYPT)
 
 #if VALID_FLAGS & PERSISTENT_FLAGS & EXPOSED
@@ -5819,7 +5820,7 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 			goto leave;
 	}
 
-	if ((rc = mdb_env_open2(env)) == MDB_SUCCESS) {
+	if ((rc = mdb_env_open2(env, flags & MDB_PREVMETA)) == MDB_SUCCESS) {
 		if (!(flags & (MDB_RDONLY|MDB_WRITEMAP))) {
 			/* Synchronous fd for meta writes. Needed even with
 			 * MDB_NOSYNC/MDB_NOMETASYNC, in case these get reset.

From d12ebb655a895e95948916304c9f17c039284d20 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 31 Jul 2017 19:07:01 +0100
Subject: [PATCH 343/504] ITS#8704 Add PREVMETA to envflags()

---
 libraries/liblmdb/mdb.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index e14b8656af..3bd93a9172 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5661,13 +5661,13 @@ mdb_enctest(const MDB_val *src, MDB_val *dst, const MDB_val *key, int encdec)
 static int ESECT
 mdb_env_envflags(MDB_env *env)
 {
-	static const char names[] = "ace" "fhi" "lmn" "rst" "w";
+	static const char names[] = "ace" "fhi" "lmn" "rst" "vw";
 	static const unsigned f[] = {
 		/*a*/ MDB_MAPASYNC, /*c*/ MDB_REMAP_CHUNKS, /*e*/ MDB_ENCRYPT,
 		/*f*/ MDB_FIXEDMAP, /*h*/ MDB_NORDAHEAD,    /*i*/ MDB_NOMEMINIT,
 		/*l*/ MDB_NOLOCK,   /*m*/ MDB_NOMETASYNC,   /*n*/ MDB_NOSUBDIR,
 		/*r*/ MDB_RDONLY,   /*s*/ MDB_NOSYNC,       /*t*/ MDB_NOTLS,
-		/*w*/ MDB_WRITEMAP,
+		/*v*/ MDB_PREVMETA, /*w*/ MDB_WRITEMAP,
 	};
 	unsigned flags = 0;
 	const char *s, *opts = getenv("LMDB_FLAGS");

From 789c71c4c99dd60857b9a481ce1de00c97d6efe3 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 31 Jul 2017 19:15:21 +0100
Subject: [PATCH 344/504] ITS#8704 Add MDB_PREVMETA support to tools

---
 libraries/liblmdb/mdb_copy.1 | 8 +++++++-
 libraries/liblmdb/mdb_copy.c | 4 +++-
 libraries/liblmdb/mdb_dump.1 | 8 +++++++-
 libraries/liblmdb/mdb_dump.c | 6 +++++-
 libraries/liblmdb/mdb_stat.1 | 8 +++++++-
 libraries/liblmdb/mdb_stat.c | 6 +++++-
 6 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/libraries/liblmdb/mdb_copy.1 b/libraries/liblmdb/mdb_copy.1
index 3b2af35aeb..edcb9a1659 100644
--- a/libraries/liblmdb/mdb_copy.1
+++ b/libraries/liblmdb/mdb_copy.1
@@ -1,4 +1,4 @@
-.TH MDB_COPY 1 "2014/07/01" "LMDB 0.9.14"
+.TH MDB_COPY 1 "2017/07/31" "LMDB 0.9.90"
 .\" Copyright 2012-2017 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
@@ -11,6 +11,8 @@ mdb_copy \- LMDB environment copy tool
 .BR \-c ]
 [\c
 .BR \-n ]
+[\c
+.BR \-v ]
 .B srcpath
 [\c
 .BR dstpath ]
@@ -40,6 +42,10 @@ Currently it fails if the environment has suffered a page leak.
 .TP
 .BR \-n
 Open LDMB environment(s) which do not use subdirectories.
+.TP
+.BR \-v
+Use the previous environment state instead of the latest state.
+This may be useful if the latest state has been corrupted.
 
 .SH DIAGNOSTICS
 Exit status is zero if no errors occur.
diff --git a/libraries/liblmdb/mdb_copy.c b/libraries/liblmdb/mdb_copy.c
index c6ec71c7e6..f156a545bd 100644
--- a/libraries/liblmdb/mdb_copy.c
+++ b/libraries/liblmdb/mdb_copy.c
@@ -38,6 +38,8 @@ int main(int argc,char * argv[])
 	for (; argc > 1 && argv[1][0] == '-'; argc--, argv++) {
 		if (argv[1][1] == 'n' && argv[1][2] == '\0')
 			flags |= MDB_NOSUBDIR;
+		else if (argv[1][1] == 'v' && argv[1][2] == '\0')
+			flags |= MDB_PREVMETA;
 		else if (argv[1][1] == 'c' && argv[1][2] == '\0')
 			cpflags |= MDB_CP_COMPACT;
 		else if (argv[1][1] == 'V' && argv[1][2] == '\0') {
@@ -48,7 +50,7 @@ int main(int argc,char * argv[])
 	}
 
 	if (argc<2 || argc>3) {
-		fprintf(stderr, "usage: %s [-V] [-c] [-n] srcpath [dstpath]\n", progname);
+		fprintf(stderr, "usage: %s [-V] [-c] [-n] [-v] srcpath [dstpath]\n", progname);
 		exit(EXIT_FAILURE);
 	}
 
diff --git a/libraries/liblmdb/mdb_dump.1 b/libraries/liblmdb/mdb_dump.1
index c4bfcb382b..37ccae42f2 100644
--- a/libraries/liblmdb/mdb_dump.1
+++ b/libraries/liblmdb/mdb_dump.1
@@ -1,4 +1,4 @@
-.TH MDB_DUMP 1 "2015/09/30" "LMDB 0.9.17"
+.TH MDB_DUMP 1 "2017/07/31" "LMDB 0.9.90"
 .\" Copyright 2014-2017 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
@@ -14,6 +14,8 @@ mdb_dump \- LMDB environment export tool
 [\c
 .BR \-n ]
 [\c
+.BR \-v ]
+[\c
 .BR \-p ]
 [\c
 .BR \-a \ |
@@ -42,6 +44,10 @@ names will be listed, no data will be output.
 .BR \-n
 Dump an LMDB database which does not use subdirectories.
 .TP
+.BR \-v
+Use the previous environment state instead of the latest state.
+This may be useful if the latest state has been corrupted.
+.TP
 .BR \-p
 If characters in either the key or data items are printing characters (as
 defined by isprint(3)), output them directly. This option permits users to
diff --git a/libraries/liblmdb/mdb_dump.c b/libraries/liblmdb/mdb_dump.c
index 867284d399..05402bc18d 100644
--- a/libraries/liblmdb/mdb_dump.c
+++ b/libraries/liblmdb/mdb_dump.c
@@ -151,7 +151,7 @@ static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name)
 
 static void usage(char *prog)
 {
-	fprintf(stderr, "usage: %s [-V] [-f output] [-l] [-n] [-p] [-a|-s subdb] dbpath\n", prog);
+	fprintf(stderr, "usage: %s [-V] [-f output] [-l] [-n] [-p] [-v] [-a|-s subdb] dbpath\n", prog);
 	exit(EXIT_FAILURE);
 }
 
@@ -175,6 +175,7 @@ int main(int argc, char *argv[])
 	 * -n: use NOSUBDIR flag on env_open
 	 * -p: use printable characters
 	 * -f: write to file instead of stdout
+	 * -v: use previous metapage
 	 * -V: print version and exit
 	 * (default) dump only the main DB
 	 */
@@ -202,6 +203,9 @@ int main(int argc, char *argv[])
 		case 'n':
 			envflags |= MDB_NOSUBDIR;
 			break;
+		case 'v':
+			envflags |= MDB_PREVMETA;
+			break;
 		case 'p':
 			mode |= PRINT;
 			break;
diff --git a/libraries/liblmdb/mdb_stat.1 b/libraries/liblmdb/mdb_stat.1
index 3520c92ae3..236019dc6e 100644
--- a/libraries/liblmdb/mdb_stat.1
+++ b/libraries/liblmdb/mdb_stat.1
@@ -1,4 +1,4 @@
-.TH MDB_STAT 1 "2015/09/30" "LMDB 0.9.17"
+.TH MDB_STAT 1 "2017/07/31" "LMDB 0.9.90"
 .\" Copyright 2012-2017 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
@@ -14,6 +14,8 @@ mdb_stat \- LMDB environment status tool
 [\c
 .BR \-n ]
 [\c
+.BR \-v ]
+[\c
 .BR \-r [ r ]]
 [\c
 .BR \-a \ |
@@ -39,6 +41,10 @@ If \fB\-fff\fP is given, display the full list of page IDs in the freelist.
 .BR \-n
 Display the status of an LMDB database which does not use subdirectories.
 .TP
+.BR \-v
+Use the previous environment state instead of the latest state.
+This may be useful if the latest state has been corrupted.
+.TP
 .BR \-r
 Display information about the environment reader table.
 Shows the process ID, thread ID, and transaction ID for each active
diff --git a/libraries/liblmdb/mdb_stat.c b/libraries/liblmdb/mdb_stat.c
index 0f0fa847f3..15145b8456 100644
--- a/libraries/liblmdb/mdb_stat.c
+++ b/libraries/liblmdb/mdb_stat.c
@@ -34,7 +34,7 @@ static void prstat(MDB_stat *ms)
 
 static void usage(char *prog)
 {
-	fprintf(stderr, "usage: %s [-V] [-n] [-e] [-r[r]] [-f[f[f]]] [-a|-s subdb] dbpath\n", prog);
+	fprintf(stderr, "usage: %s [-V] [-n] [-e] [-r[r]] [-f[f[f]]] [-v] [-a|-s subdb] dbpath\n", prog);
 	exit(EXIT_FAILURE);
 }
 
@@ -61,6 +61,7 @@ int main(int argc, char *argv[])
 	 * -f: print freelist info
 	 * -r: print reader info
 	 * -n: use NOSUBDIR flag on env_open
+	 * -v: use previous metapage
 	 * -V: print version and exit
 	 * (default) print stat of only the main DB
 	 */
@@ -84,6 +85,9 @@ int main(int argc, char *argv[])
 		case 'n':
 			envflags |= MDB_NOSUBDIR;
 			break;
+		case 'v':
+			envflags |= MDB_PREVMETA;
+			break;
 		case 'r':
 			rdrinfo++;
 			break;

From 73d75c6215d8e6cc91d0f16f1cf7c73b6e97ab93 Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sun, 6 Aug 2017 16:15:51 +0200
Subject: [PATCH 345/504] Drop always-false MDB_WRITEMAP test when
 IS_DIRTY_NW()

---
 libraries/liblmdb/mdb.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 3bd93a9172..7bfa256704 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -6992,8 +6992,7 @@ mdb_ovpage_free(MDB_cursor *mc, MDB_page *mp)
 			}
 		}
 		txn->mt_dirty_room++;
-		if (!(env->me_flags & MDB_WRITEMAP))
-			mdb_dpage_free(env, mp);
+		mdb_dpage_free(env, mp);
 release:
 		/* Insert in me_pghead */
 		mop = env->me_pghead;

From 16839f989396355c8126f6a14fe54657c97ed46f Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sun, 6 Aug 2017 16:18:06 +0200
Subject: [PATCH 346/504] Unref ovpages when deleting them

---
 libraries/liblmdb/mdb.c | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 7bfa256704..217e203839 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -2234,10 +2234,27 @@ mdb_cursor_unref(MDB_cursor *mc)
 	 ? mdb_cursor_unref(mc) \
 	 : (void)0)
 
+/* Unref ovpage \b omp in \b mc and tracked cursors */
+static void
+mdb_ovpage_unref_all(MDB_cursor *mc, MDB_page *omp)
+{
+	MDB_txn *txn = mc->mc_txn;
+	MDB_cursor *next = txn->mt_cursors[mc->mc_dbi];
+	for (;; mc = next, next = mc->mc_next) {
+		if (MC_OVPG(mc) == omp) {
+			mdb_page_unref(mc->mc_txn, omp);
+			MC_SET_OVPG(mc, NULL);
+		}
+		if (next == NULL)
+			break;
+	}
+}
+
 #else
 #define MDB_REMAPPING(flags)	0
 #define MDB_PAGE_UNREF(txn, mp)
 #define MDB_CURSOR_UNREF(mc, force) ((void)0)
+#define mdb_ovpage_unref_all(mc, omp, pgno) ((void)0)
 #endif /* MDB_RPAGE_CACHE */
 
 /** Loosen or free a single page.
@@ -6943,6 +6960,7 @@ mdb_ovpage_free(MDB_cursor *mc, MDB_page *mp)
 	unsigned x = 0, ovpages = mp->mp_pages;
 	MDB_env *env = txn->mt_env;
 	MDB_ID pn = pg << 1;
+	MDB_page *freeme = NULL;
 	int rc;
 
 	DPRINTF(("free ov page %"Yu" (%d)", pg, ovpages));
@@ -6992,7 +7010,7 @@ mdb_ovpage_free(MDB_cursor *mc, MDB_page *mp)
 			}
 		}
 		txn->mt_dirty_room++;
-		mdb_dpage_free(env, mp);
+		freeme = mp;
 release:
 		/* Insert in me_pghead */
 		mop = env->me_pghead;
@@ -7008,6 +7026,12 @@ release:
 			return rc;
 	}
 	mc->mc_db->md_overflow_pages -= ovpages;
+
+	if (MDB_REMAPPING(env->me_flags))
+		mdb_ovpage_unref_all(mc, mp);
+	if (freeme)
+		mdb_dpage_free(env, freeme);
+
 	return 0;
 }
 

From 3585a1eb977326c7e178c53f4eef1fdc81b46e63 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sat, 12 Aug 2017 11:16:45 +0100
Subject: [PATCH 347/504] ITS#8704 Fix PREVMETA, rename to PREVSNAPSHOT

and enforce exclusive access to environment. Also fix txn_begin/pick_meta
to use correct meta page, and reset the flag after successful commit.
---
 libraries/liblmdb/lmdb.h     | 12 +++++++-----
 libraries/liblmdb/mdb.c      | 32 ++++++++++++++++++++++++--------
 libraries/liblmdb/mdb_copy.c |  2 +-
 libraries/liblmdb/mdb_dump.c |  4 ++--
 libraries/liblmdb/mdb_stat.c |  4 ++--
 5 files changed, 36 insertions(+), 18 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 3a6be034a4..f9f847f1ce 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -354,8 +354,8 @@ typedef void (MDB_enc_func)(const MDB_val *src, MDB_val *dst, const MDB_val *key
 #define MDB_NORDAHEAD	0x800000
 	/** don't initialize malloc'd memory before writing to datafile */
 #define MDB_NOMEMINIT	0x1000000
-	/** use the previous meta page rather than the latest one */
-#define MDB_PREVMETA	0x2000000
+	/** use the previous snapshot rather than the latest one */
+#define MDB_PREVSNAPSHOT	0x2000000
 	/** don't use a single mmap, remap individual chunks (needs MDB_RPAGE_CACHE) */
 #define MDB_REMAP_CHUNKS	0x4000000
 /** @} */
@@ -672,10 +672,12 @@ int  mdb_env_create(MDB_env **env);
 	 *		caller is expected to overwrite all of the memory that was
 	 *		reserved in that case.
 	 *		This flag may be changed at any time using #mdb_env_set_flags().
-	 *	<li>#MDB_PREVMETA
-	 *		Open the environment with the previous meta page rather than the latest
+	 *	<li>#MDB_PREVSNAPSHOT
+	 *		Open the environment with the previous snapshot rather than the latest
 	 *		one. This loses the latest transaction, but may help work around some
-	 *		types of corruption.
+	 *		types of corruption. If opened with write access, this must be the
+	 *		only process using the environment. This flag is automatically reset
+	 *		after a write transaction is successfully committed.
 	 * </ul>
 	 * @param[in] mode The UNIX permissions to set on created files and semaphores.
 	 * This parameter is ignored on Windows.
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 217e203839..4b77458806 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -4047,6 +4047,8 @@ done:
 	return MDB_SUCCESS;
 }
 
+static int ESECT mdb_env_share_locks(MDB_env *env, int *excl);
+
 int
 mdb_txn_commit(MDB_txn *txn)
 {
@@ -4274,6 +4276,15 @@ mdb_txn_commit(MDB_txn *txn)
 	if ((rc = mdb_env_write_meta(txn)))
 		goto fail;
 	end_mode = MDB_END_COMMITTED|MDB_END_UPDATE;
+	if (env->me_flags & MDB_PREVSNAPSHOT) {
+		if (!(env->me_flags & MDB_NOLOCK)) {
+			int excl;
+			rc = mdb_env_share_locks(env, &excl);
+			if (rc)
+				goto fail;
+		}
+		env->me_flags ^= MDB_PREVSNAPSHOT;
+	}
 
 done:
 	mdb_txn_end(txn, end_mode);
@@ -4564,7 +4575,8 @@ static MDB_meta *
 mdb_env_pick_meta(const MDB_env *env)
 {
 	MDB_meta *const *metas = env->me_metas;
-	return metas[ metas[0]->mm_txnid < metas[1]->mm_txnid ];
+	return metas[ (metas[0]->mm_txnid < metas[1]->mm_txnid) ^
+		((env->me_flags & MDB_PREVSNAPSHOT) != 0) ];
 }
 
 int ESECT
@@ -5139,6 +5151,9 @@ mdb_env_open2(MDB_env *env, int prev)
 #endif
 	env->me_maxpg = env->me_mapsize / env->me_psize;
 
+	if (env->me_txns)
+		env->me_txns->mti_txnid = meta.mm_txnid;
+
 #if MDB_DEBUG
 	{
 		MDB_meta *meta = mdb_env_pick_meta(env);
@@ -5238,9 +5253,6 @@ static int ESECT
 mdb_env_share_locks(MDB_env *env, int *excl)
 {
 	int rc = 0;
-	MDB_meta *meta = mdb_env_pick_meta(env);
-
-	env->me_txns->mti_txnid = meta->mm_txnid;
 
 #ifdef _WIN32
 	{
@@ -5684,7 +5696,7 @@ mdb_env_envflags(MDB_env *env)
 		/*f*/ MDB_FIXEDMAP, /*h*/ MDB_NORDAHEAD,    /*i*/ MDB_NOMEMINIT,
 		/*l*/ MDB_NOLOCK,   /*m*/ MDB_NOMETASYNC,   /*n*/ MDB_NOSUBDIR,
 		/*r*/ MDB_RDONLY,   /*s*/ MDB_NOSYNC,       /*t*/ MDB_NOTLS,
-		/*v*/ MDB_PREVMETA, /*w*/ MDB_WRITEMAP,
+		/*v*/ MDB_PREVSNAPSHOT, /*w*/ MDB_WRITEMAP,
 	};
 	unsigned flags = 0;
 	const char *s, *opts = getenv("LMDB_FLAGS");
@@ -5724,7 +5736,7 @@ mdb_env_envflags(MDB_env *env)
 	 */
 #define	CHANGEABLE	(MDB_NOSYNC|MDB_NOMETASYNC|MDB_MAPASYNC|MDB_NOMEMINIT)
 #define	CHANGELESS	(MDB_FIXEDMAP|MDB_NOSUBDIR|MDB_RDONLY| \
-	MDB_WRITEMAP|MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD|MDB_PREVMETA|MDB_REMAP_CHUNKS)
+	MDB_WRITEMAP|MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD|MDB_PREVSNAPSHOT|MDB_REMAP_CHUNKS)
 #define EXPOSED		(CHANGEABLE|CHANGELESS | MDB_ENCRYPT)
 
 #if VALID_FLAGS & PERSISTENT_FLAGS & EXPOSED
@@ -5823,6 +5835,10 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 		rc = mdb_env_setup_locks(env, &fname, mode, &excl);
 		if (rc)
 			goto leave;
+		if ((flags & MDB_PREVSNAPSHOT) && !excl) {
+			rc = EAGAIN;
+			goto leave;
+		}
 	}
 
 	rc = mdb_fopen(env, &fname,
@@ -5837,7 +5853,7 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 			goto leave;
 	}
 
-	if ((rc = mdb_env_open2(env, flags & MDB_PREVMETA)) == MDB_SUCCESS) {
+	if ((rc = mdb_env_open2(env, flags & MDB_PREVSNAPSHOT)) == MDB_SUCCESS) {
 		if (!(flags & (MDB_RDONLY|MDB_WRITEMAP))) {
 			/* Synchronous fd for meta writes. Needed even with
 			 * MDB_NOSYNC/MDB_NOMETASYNC, in case these get reset.
@@ -5847,7 +5863,7 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 				goto leave;
 		}
 		DPRINTF(("opened dbenv %p", (void *) env));
-		if (excl > 0) {
+		if (excl > 0 && !(flags & MDB_PREVSNAPSHOT)) {
 			rc = mdb_env_share_locks(env, &excl);
 			if (rc)
 				goto leave;
diff --git a/libraries/liblmdb/mdb_copy.c b/libraries/liblmdb/mdb_copy.c
index f156a545bd..fd7268dc4d 100644
--- a/libraries/liblmdb/mdb_copy.c
+++ b/libraries/liblmdb/mdb_copy.c
@@ -39,7 +39,7 @@ int main(int argc,char * argv[])
 		if (argv[1][1] == 'n' && argv[1][2] == '\0')
 			flags |= MDB_NOSUBDIR;
 		else if (argv[1][1] == 'v' && argv[1][2] == '\0')
-			flags |= MDB_PREVMETA;
+			flags |= MDB_PREVSNAPSHOT;
 		else if (argv[1][1] == 'c' && argv[1][2] == '\0')
 			cpflags |= MDB_CP_COMPACT;
 		else if (argv[1][1] == 'V' && argv[1][2] == '\0') {
diff --git a/libraries/liblmdb/mdb_dump.c b/libraries/liblmdb/mdb_dump.c
index 05402bc18d..77c2a5a358 100644
--- a/libraries/liblmdb/mdb_dump.c
+++ b/libraries/liblmdb/mdb_dump.c
@@ -175,7 +175,7 @@ int main(int argc, char *argv[])
 	 * -n: use NOSUBDIR flag on env_open
 	 * -p: use printable characters
 	 * -f: write to file instead of stdout
-	 * -v: use previous metapage
+	 * -v: use previous snapshot
 	 * -V: print version and exit
 	 * (default) dump only the main DB
 	 */
@@ -204,7 +204,7 @@ int main(int argc, char *argv[])
 			envflags |= MDB_NOSUBDIR;
 			break;
 		case 'v':
-			envflags |= MDB_PREVMETA;
+			envflags |= MDB_PREVSNAPSHOT;
 			break;
 		case 'p':
 			mode |= PRINT;
diff --git a/libraries/liblmdb/mdb_stat.c b/libraries/liblmdb/mdb_stat.c
index 15145b8456..c67ed6021a 100644
--- a/libraries/liblmdb/mdb_stat.c
+++ b/libraries/liblmdb/mdb_stat.c
@@ -61,7 +61,7 @@ int main(int argc, char *argv[])
 	 * -f: print freelist info
 	 * -r: print reader info
 	 * -n: use NOSUBDIR flag on env_open
-	 * -v: use previous metapage
+	 * -v: use previous snapshot
 	 * -V: print version and exit
 	 * (default) print stat of only the main DB
 	 */
@@ -86,7 +86,7 @@ int main(int argc, char *argv[])
 			envflags |= MDB_NOSUBDIR;
 			break;
 		case 'v':
-			envflags |= MDB_PREVMETA;
+			envflags |= MDB_PREVSNAPSHOT;
 			break;
 		case 'r':
 			rdrinfo++;

From e4cf950264d740c163fff34f2f9017ace1847d09 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 31 Aug 2017 15:11:54 +0100
Subject: [PATCH 348/504] mdb_dbi_open tweak

Always save the terminating NUL of the incoming DB name
Stop using the old mdb_open/mdb_close function names
 .. should consider renaming mdb_stat/mdb_drop to mdb_dbi_... too.
---
 libraries/liblmdb/mdb.c      | 12 ++++++------
 libraries/liblmdb/mdb_dump.c | 19 +++++++------------
 libraries/liblmdb/mdb_load.c |  4 ++--
 libraries/liblmdb/mdb_stat.c | 19 +++++++------------
 4 files changed, 22 insertions(+), 32 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 4b77458806..8f2a18ccd6 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -11110,7 +11110,7 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db
 	int rc, dbflag, exact;
 	unsigned int unused = 0, seq;
 	char *namedup;
-	size_t len;
+	size_t size;
 
 	if (flags & ~VALID_FLAGS)
 		return EINVAL;
@@ -11137,15 +11137,15 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db
 	}
 
 	/* Is the DB already open? */
-	len = strlen(name);
+	size = strlen(name) + 1;
 	for (i=CORE_DBS; i<txn->mt_numdbs; i++) {
 		if (!txn->mt_dbxs[i].md_name.mv_size) {
 			/* Remember this free slot */
 			if (!unused) unused = i;
 			continue;
 		}
-		if (len == txn->mt_dbxs[i].md_name.mv_size &&
-			!strncmp(name, txn->mt_dbxs[i].md_name.mv_data, len)) {
+		if (size == txn->mt_dbxs[i].md_name.mv_size &&
+			!strcmp(name, txn->mt_dbxs[i].md_name.mv_data)) {
 			*dbi = i;
 			return MDB_SUCCESS;
 		}
@@ -11162,7 +11162,7 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db
 	/* Find the DB info */
 	dbflag = DB_NEW|DB_VALID|DB_USRVALID;
 	exact = 0;
-	key.mv_size = len;
+	key.mv_size = size;
 	key.mv_data = (void *)name;
 	mdb_cursor_init(&mc, txn, MAIN_DBI, NULL);
 	rc = mdb_cursor_set(&mc, &key, &data, MDB_SET, &exact);
@@ -11197,7 +11197,7 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db
 		/* Got info, register DBI in this txn */
 		unsigned int slot = unused ? unused : txn->mt_numdbs;
 		txn->mt_dbxs[slot].md_name.mv_data = namedup;
-		txn->mt_dbxs[slot].md_name.mv_size = len;
+		txn->mt_dbxs[slot].md_name.mv_size = size;
 		txn->mt_dbxs[slot].md_rel = NULL;
 		txn->mt_dbflags[slot] = dbflag;
 		/* txn-> and env-> are the same in read txns, use
diff --git a/libraries/liblmdb/mdb_dump.c b/libraries/liblmdb/mdb_dump.c
index 77c2a5a358..57e79837df 100644
--- a/libraries/liblmdb/mdb_dump.c
+++ b/libraries/liblmdb/mdb_dump.c
@@ -254,9 +254,9 @@ int main(int argc, char *argv[])
 		goto env_close;
 	}
 
-	rc = mdb_open(txn, subname, 0, &dbi);
+	rc = mdb_dbi_open(txn, subname, 0, &dbi);
 	if (rc) {
-		fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
+		fprintf(stderr, "mdb_dbi_open failed, error %d %s\n", rc, mdb_strerror(rc));
 		goto txn_abort;
 	}
 
@@ -271,27 +271,22 @@ int main(int argc, char *argv[])
 			goto txn_abort;
 		}
 		while ((rc = mdb_cursor_get(cursor, &key, NULL, MDB_NEXT_NODUP)) == 0) {
-			char *str;
 			MDB_dbi db2;
-			if (memchr(key.mv_data, '\0', key.mv_size))
+			if (memchr(key.mv_data, '\0', key.mv_size-1) || ((char *)key.mv_data)[key.mv_size=1] != '\0')
 				continue;
 			count++;
-			str = malloc(key.mv_size+1);
-			memcpy(str, key.mv_data, key.mv_size);
-			str[key.mv_size] = '\0';
-			rc = mdb_open(txn, str, 0, &db2);
+			rc = mdb_dbi_open(txn, key.mv_data, 0, &db2);
 			if (rc == MDB_SUCCESS) {
 				if (list) {
-					printf("%s\n", str);
+					printf("%s\n", (char *)key.mv_data);
 					list++;
 				} else {
-					rc = dumpit(txn, db2, str);
+					rc = dumpit(txn, db2, key.mv_data);
 					if (rc)
 						break;
 				}
-				mdb_close(env, db2);
+				mdb_dbi_close(env, db2);
 			}
-			free(str);
 			if (rc) continue;
 		}
 		mdb_cursor_close(cursor);
diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c
index 417fad2e3a..7888144a7e 100644
--- a/libraries/liblmdb/mdb_load.c
+++ b/libraries/liblmdb/mdb_load.c
@@ -384,9 +384,9 @@ int main(int argc, char *argv[])
 			goto env_close;
 		}
 
-		rc = mdb_open(txn, subname, flags|MDB_CREATE, &dbi);
+		rc = mdb_dbi_open(txn, subname, flags|MDB_CREATE, &dbi);
 		if (rc) {
-			fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
+			fprintf(stderr, "mdb_dbi_open failed, error %d %s\n", rc, mdb_strerror(rc));
 			goto txn_abort;
 		}
 
diff --git a/libraries/liblmdb/mdb_stat.c b/libraries/liblmdb/mdb_stat.c
index c67ed6021a..a94f5cd9fa 100644
--- a/libraries/liblmdb/mdb_stat.c
+++ b/libraries/liblmdb/mdb_stat.c
@@ -204,9 +204,9 @@ int main(int argc, char *argv[])
 		printf("  Free pages: %"Yu"\n", pages);
 	}
 
-	rc = mdb_open(txn, subname, 0, &dbi);
+	rc = mdb_dbi_open(txn, subname, 0, &dbi);
 	if (rc) {
-		fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
+		fprintf(stderr, "mdb_dbi_open failed, error %d %s\n", rc, mdb_strerror(rc));
 		goto txn_abort;
 	}
 
@@ -228,17 +228,12 @@ int main(int argc, char *argv[])
 			goto txn_abort;
 		}
 		while ((rc = mdb_cursor_get(cursor, &key, NULL, MDB_NEXT_NODUP)) == 0) {
-			char *str;
 			MDB_dbi db2;
-			if (memchr(key.mv_data, '\0', key.mv_size))
+			if (memchr(key.mv_data, '\0', key.mv_size-1) || ((char *)key.mv_data)[key.mv_size-1] != '\0')
 				continue;
-			str = malloc(key.mv_size+1);
-			memcpy(str, key.mv_data, key.mv_size);
-			str[key.mv_size] = '\0';
-			rc = mdb_open(txn, str, 0, &db2);
+			rc = mdb_dbi_open(txn, key.mv_data, 0, &db2);
 			if (rc == MDB_SUCCESS)
-				printf("Status of %s\n", str);
-			free(str);
+				printf("Status of %s\n", (char *)key.mv_data);
 			if (rc) continue;
 			rc = mdb_stat(txn, db2, &mst);
 			if (rc) {
@@ -246,7 +241,7 @@ int main(int argc, char *argv[])
 				goto txn_abort;
 			}
 			prstat(&mst);
-			mdb_close(env, db2);
+			mdb_dbi_close(env, db2);
 		}
 		mdb_cursor_close(cursor);
 	}
@@ -254,7 +249,7 @@ int main(int argc, char *argv[])
 	if (rc == MDB_NOTFOUND)
 		rc = MDB_SUCCESS;
 
-	mdb_close(env, dbi);
+	mdb_dbi_close(env, dbi);
 txn_abort:
 	mdb_txn_abort(txn);
 env_close:

From 27c8cee72d9c0c32118d6a487c0b74519b815303 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 31 Aug 2017 15:44:31 +0100
Subject: [PATCH 349/504] Add mdb_cursor_is_db()

Return 1 if the cursor is pointing to a named DB record
---
 libraries/liblmdb/lmdb.h     |  7 +++++++
 libraries/liblmdb/mdb.c      | 12 ++++++++++++
 libraries/liblmdb/mdb_dump.c |  2 +-
 libraries/liblmdb/mdb_stat.c |  2 +-
 4 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index f9f847f1ce..d6a422d7cb 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -1506,6 +1506,13 @@ MDB_txn *mdb_cursor_txn(MDB_cursor *cursor);
 	 */
 MDB_dbi mdb_cursor_dbi(MDB_cursor *cursor);
 
+	/** @brief Check if the cursor is pointing to a named database record.
+	 *
+	 * @param[in] cursor A cursor handle returned by #mdb_cursor_open()
+	 * @return 1 if current record is a named database, 0 otherwise.
+	 */
+int mdb_cursor_is_db(MDB_cursor *cursor);
+
 	/** @brief Retrieve by cursor.
 	 *
 	 * This function retrieves key/data pairs from the database. The address and length
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 8f2a18ccd6..1eebb153f2 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -9087,6 +9087,18 @@ mdb_cursor_dbi(MDB_cursor *mc)
 	return mc->mc_dbi;
 }
 
+int
+mdb_cursor_is_db(MDB_cursor *mc)
+{
+
+	if (mc && (mc->mc_flags & C_INITIALIZED) && mc->mc_snum) {
+		MDB_node *leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
+		if ((leaf->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_SUBDATA)
+			return 1;
+	}
+	return 0;
+}
+
 /** Replace the key for a branch node with a new key.
  * Set #MDB_TXN_ERROR on failure.
  * @param[in] mc Cursor pointing to the node to operate on.
diff --git a/libraries/liblmdb/mdb_dump.c b/libraries/liblmdb/mdb_dump.c
index 57e79837df..fa1ca2966b 100644
--- a/libraries/liblmdb/mdb_dump.c
+++ b/libraries/liblmdb/mdb_dump.c
@@ -272,7 +272,7 @@ int main(int argc, char *argv[])
 		}
 		while ((rc = mdb_cursor_get(cursor, &key, NULL, MDB_NEXT_NODUP)) == 0) {
 			MDB_dbi db2;
-			if (memchr(key.mv_data, '\0', key.mv_size-1) || ((char *)key.mv_data)[key.mv_size=1] != '\0')
+			if (!mdb_cursor_is_db(cursor))
 				continue;
 			count++;
 			rc = mdb_dbi_open(txn, key.mv_data, 0, &db2);
diff --git a/libraries/liblmdb/mdb_stat.c b/libraries/liblmdb/mdb_stat.c
index a94f5cd9fa..f274a4666c 100644
--- a/libraries/liblmdb/mdb_stat.c
+++ b/libraries/liblmdb/mdb_stat.c
@@ -229,7 +229,7 @@ int main(int argc, char *argv[])
 		}
 		while ((rc = mdb_cursor_get(cursor, &key, NULL, MDB_NEXT_NODUP)) == 0) {
 			MDB_dbi db2;
-			if (memchr(key.mv_data, '\0', key.mv_size-1) || ((char *)key.mv_data)[key.mv_size-1] != '\0')
+			if (!mdb_cursor_is_db(cursor))
 				continue;
 			rc = mdb_dbi_open(txn, key.mv_data, 0, &db2);
 			if (rc == MDB_SUCCESS)

From 05d5a9d3f6c1b808380c082d84b1db7e8472045d Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Fri, 1 Sep 2017 13:16:37 +0100
Subject: [PATCH 350/504] Doc tweaks

Note about DB names being C strings
---
 libraries/liblmdb/lmdb.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index d6a422d7cb..bc1e2684ba 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -224,7 +224,7 @@ typedef int mdb_filehandle_t;
 
 /** @defgroup mdb LMDB API
  *	@{
- *	@brief OpenLDAP Lightning Memory-Mapped Database Manager
+ *	@brief Symas Lightning Memory-Mapped Database Manager
  */
 /** @defgroup Version Version Macros
  *	@{
@@ -1161,6 +1161,8 @@ int  mdb_txn_renew(MDB_txn *txn);
 	 * To use named databases (with name != NULL), #mdb_env_set_maxdbs()
 	 * must be called before opening the environment.  Database names are
 	 * keys in the unnamed database, and may be read but not written.
+	 * @note Names are C strings and stored with their NUL terminator included.
+	 * In LMDB 0.9 the NUL terminator was omitted.
 	 *
 	 * @param[in] txn A transaction handle returned by #mdb_txn_begin()
 	 * @param[in] name The name of the database to open. If only a single

From 0e91bde589084e123cf16875866a4ab9c5044d09 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 6 Sep 2017 21:15:48 +0100
Subject: [PATCH 351/504] ITS#8722 fix FIRST_DUP/LAST_DUP cursor bounds check

---
 libraries/liblmdb/mdb.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 1eebb153f2..9c0b3ca450 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7799,6 +7799,11 @@ fetchm:
 			rc = MDB_INCOMPATIBLE;
 			break;
 		}
+		if (mc->mc_ki[mc->mc_top] >= NUMKEYS(mc->mc_pg[mc->mc_top])) {
+			mc->mc_ki[mc->mc_top] = NUMKEYS(mc->mc_pg[mc->mc_top]);
+			rc = MDB_NOTFOUND;
+			break;
+		}
 		{
 			MDB_node *leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
 			if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) {
@@ -8448,6 +8453,7 @@ mdb_cursor_del(MDB_cursor *mc, unsigned int flags)
 						if (!(m2->mc_flags & C_INITIALIZED)) continue;
 						if (m2->mc_pg[mc->mc_top] == mp) {
 							MDB_node *n2 = leaf;
+							if (m2->mc_ki[mc->mc_top] >= NUMKEYS(mp)) continue;
 							if (m2->mc_ki[mc->mc_top] != mc->mc_ki[mc->mc_top]) {
 								n2 = NODEPTR(mp, m2->mc_ki[mc->mc_top]);
 								if (n2->mn_flags & F_SUBDATA) continue;

From 2b397001a8cc451b9e09bda205c7191a31b781a7 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sat, 9 Sep 2017 15:08:03 +0100
Subject: [PATCH 352/504] ITS#8728 fix MDB_VL32 freeing overflow page

---
 libraries/liblmdb/mdb.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 9c0b3ca450..6287590bb4 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7041,6 +7041,8 @@ release:
 		if (rc)
 			return rc;
 	}
+	if (MC_OVPG(mc) == mp)
+		MC_SET_OVPG(mc, NULL);
 	mc->mc_db->md_overflow_pages -= ovpages;
 
 	if (MDB_REMAPPING(env->me_flags))

From 5601781247bfd132998ed03fa6e24e7035ae383a Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Sun, 10 Sep 2017 23:59:35 +0200
Subject: [PATCH 353/504] XCURSOR_REFRESH() fixups/cleanup

* Check NUMKEYS(), similar to f34b61f9471d1c03fe0517b9d817c50c920e378a
  "ITS#8722 fix FIRST_DUP/LAST_DUP cursor bounds check".
* Move XCURSOR_INITED() into XCURSOR_REFRESH().  This adds a check in
  mdb_cursor_put, below /* converted, write the original data first */.
* Factor mc_ki[] out to XCURSOR_REFRESH().
* Replace an mc_pg[] with mp which is equal (mdb_cursor_del0).
---
 libraries/liblmdb/mdb.c | 43 ++++++++++++++++++++---------------------
 1 file changed, 21 insertions(+), 22 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 6287590bb4..7f67bbb887 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1527,17 +1527,19 @@ typedef struct MDB_xcursor {
 	unsigned char mx_dbflag;
 } MDB_xcursor;
 
-	/** Check if there is an inited xcursor, so #XCURSOR_REFRESH() is proper */
+	/** Check if there is an inited xcursor */
 #define XCURSOR_INITED(mc) \
 	((mc)->mc_xcursor && ((mc)->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED))
 
-	/** Update sub-page pointer, if any, in \b mc->mc_xcursor.  Needed
+	/** Update the xcursor's sub-page pointer, if any, in \b mc.  Needed
 	 *	when the node which contains the sub-page may have moved.  Called
-	 *	with \b mp = mc->mc_pg[mc->mc_top], \b ki = mc->mc_ki[mc->mc_top].
+	 *	with leaf page \b mp = mc->mc_pg[\b top].
 	 */
-#define XCURSOR_REFRESH(mc, mp, ki) do { \
+#define XCURSOR_REFRESH(mc, top, mp) do { \
 	MDB_page *xr_pg = (mp); \
-	MDB_node *xr_node = NODEPTR(xr_pg, ki); \
+	MDB_node *xr_node; \
+	if (!XCURSOR_INITED(mc) || (mc)->mc_ki[top] >= NUMKEYS(xr_pg)) break; \
+	xr_node = NODEPTR(xr_pg, (mc)->mc_ki[top]); \
 	if ((xr_node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) \
 		(mc)->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(xr_node); \
 } while (0)
@@ -2946,8 +2948,8 @@ done:
 			if (m2 == mc) continue;
 			if (m2->mc_pg[mc->mc_top] == mp) {
 				m2->mc_pg[mc->mc_top] = np;
-				if (XCURSOR_INITED(m2) && IS_LEAF(np))
-					XCURSOR_REFRESH(m2, np, m2->mc_ki[mc->mc_top]);
+				if (IS_LEAF(np))
+					XCURSOR_REFRESH(m2, mc->mc_top, np);
 			}
 		}
 	}
@@ -8297,8 +8299,7 @@ new_sub:
 				if (m3->mc_ki[i] >= mc->mc_ki[i] && insert_key) {
 					m3->mc_ki[i]++;
 				}
-				if (XCURSOR_INITED(m3))
-					XCURSOR_REFRESH(m3, mp, m3->mc_ki[i]);
+				XCURSOR_REFRESH(m3, i, mp);
 			}
 		}
 	}
@@ -8340,7 +8341,6 @@ put_sub:
 				MDB_xcursor *mx = mc->mc_xcursor;
 				unsigned i = mc->mc_top;
 				MDB_page *mp = mc->mc_pg[i];
-				int nkeys = NUMKEYS(mp);
 
 				for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) {
 					if (m2 == mc || m2->mc_snum < mc->mc_snum) continue;
@@ -8348,8 +8348,8 @@ put_sub:
 					if (m2->mc_pg[i] == mp) {
 						if (m2->mc_ki[i] == mc->mc_ki[i]) {
 							mdb_xcursor_init2(m2, mx, new_dupdata);
-						} else if (!insert_key && m2->mc_ki[i] < nkeys) {
-							XCURSOR_REFRESH(m2, mp, m2->mc_ki[i]);
+						} else if (!insert_key) {
+							XCURSOR_REFRESH(m2, i, mp);
 						}
 					}
 				}
@@ -9328,8 +9328,8 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft)
 					m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top];
 					m3->mc_ki[csrc->mc_top-1]++;
 				}
-				if (XCURSOR_INITED(m3) && IS_LEAF(mps))
-					XCURSOR_REFRESH(m3, m3->mc_pg[csrc->mc_top], m3->mc_ki[csrc->mc_top]);
+				if (IS_LEAF(mps))
+					XCURSOR_REFRESH(m3, csrc->mc_top, m3->mc_pg[csrc->mc_top]);
 			}
 		} else
 		/* Adding on the right, bump others down */
@@ -9350,8 +9350,8 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft)
 					} else {
 						m3->mc_ki[csrc->mc_top]--;
 					}
-					if (XCURSOR_INITED(m3) && IS_LEAF(mps))
-						XCURSOR_REFRESH(m3, m3->mc_pg[csrc->mc_top], m3->mc_ki[csrc->mc_top]);
+					if (IS_LEAF(mps))
+						XCURSOR_REFRESH(m3, csrc->mc_top, m3->mc_pg[csrc->mc_top]);
 				}
 			}
 		}
@@ -9552,8 +9552,8 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst)
 				m3->mc_ki[top-1] > csrc->mc_ki[top-1]) {
 				m3->mc_ki[top-1]--;
 			}
-			if (XCURSOR_INITED(m3) && IS_LEAF(psrc))
-				XCURSOR_REFRESH(m3, m3->mc_pg[top], m3->mc_ki[top]);
+			if (IS_LEAF(psrc))
+				XCURSOR_REFRESH(m3, top, m3->mc_pg[top]);
 		}
 	}
 	{
@@ -9816,8 +9816,7 @@ mdb_cursor_del0(MDB_cursor *mc)
 				} else if (m3->mc_ki[mc->mc_top] > ki) {
 					m3->mc_ki[mc->mc_top]--;
 				}
-				if (XCURSOR_INITED(m3))
-					XCURSOR_REFRESH(m3, m3->mc_pg[mc->mc_top], m3->mc_ki[mc->mc_top]);
+				XCURSOR_REFRESH(m3, mc->mc_top, mp);
 			}
 		}
 	}
@@ -10355,8 +10354,8 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 				m3->mc_ki[ptop] >= mc->mc_ki[ptop]) {
 				m3->mc_ki[ptop]++;
 			}
-			if (XCURSOR_INITED(m3) && IS_LEAF(mp))
-				XCURSOR_REFRESH(m3, m3->mc_pg[mc->mc_top], m3->mc_ki[mc->mc_top]);
+			if (IS_LEAF(mp))
+				XCURSOR_REFRESH(m3, mc->mc_top, m3->mc_pg[mc->mc_top]);
 		}
 	}
 	DPRINTF(("mp left: %d, rp left: %d", SIZELEFT(mp), SIZELEFT(rp)));

From b9488faae4c5d3e754bfafbc90652d2953b6cc7d Mon Sep 17 00:00:00 2001
From: Hallvard Furuseth <hallvard@openldap.org>
Date: Mon, 11 Sep 2017 00:00:14 +0200
Subject: [PATCH 354/504] Tweak ITS#8722 fix: Use XCURSOR_REFRESH()

This checks XCURSOR_INITED() and fixes the mn_flags check.
---
 libraries/liblmdb/mdb.c | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 7f67bbb887..79e96e2bbd 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -8454,13 +8454,7 @@ mdb_cursor_del(MDB_cursor *mc, unsigned int flags)
 						if (m2 == mc || m2->mc_snum < mc->mc_snum) continue;
 						if (!(m2->mc_flags & C_INITIALIZED)) continue;
 						if (m2->mc_pg[mc->mc_top] == mp) {
-							MDB_node *n2 = leaf;
-							if (m2->mc_ki[mc->mc_top] >= NUMKEYS(mp)) continue;
-							if (m2->mc_ki[mc->mc_top] != mc->mc_ki[mc->mc_top]) {
-								n2 = NODEPTR(mp, m2->mc_ki[mc->mc_top]);
-								if (n2->mn_flags & F_SUBDATA) continue;
-							}
-							m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(n2);
+							XCURSOR_REFRESH(m2, mc->mc_top, mp);
 						}
 					}
 				}

From d9ef7ace69f69306aa82998c67550dc78a19c6df Mon Sep 17 00:00:00 2001
From: Quanah Gibson-Mount <quanah@symas.com>
Date: Wed, 7 Jun 2017 13:42:51 -0700
Subject: [PATCH 355/504] ITS#8612 Fix Solaris builds with liblmdb

This patch fixes liblmdb builds on Solaris and derivatives by defining
_POSIX_PTHREAD_SEMANTICS
---
 libraries/liblmdb/mdb.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 79e96e2bbd..cecd1da6df 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -142,6 +142,10 @@ typedef SSIZE_T	ssize_t;
 /* Most platforms have posix_memalign, older may only have memalign */
 #define HAVE_MEMALIGN	1
 #include <malloc.h>
+/* On Solaris, we need the POSIX sigwait function */
+#if defined (__sun)
+# define _POSIX_PTHREAD_SEMANTICS	1
+#endif
 #endif
 
 #if !(defined(BYTE_ORDER) || defined(__BYTE_ORDER))

From cf3588baaecc835b24a0ebf653bc3de7ea370887 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 20 Sep 2017 18:38:47 +0100
Subject: [PATCH 356/504] Add -a append option to mdb_load

To allow reloading of custom-sorted DBs from mdb_dump
---
 libraries/liblmdb/mdb.c      |  4 +--
 libraries/liblmdb/mdb_load.1 |  7 +++++
 libraries/liblmdb/mdb_load.c | 52 +++++++++++++++++++++++++++++++-----
 3 files changed, 54 insertions(+), 9 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index cecd1da6df..f3aa3c07ad 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7932,7 +7932,7 @@ mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data,
 
 	dkey.mv_size = 0;
 
-	if (flags == MDB_CURRENT) {
+	if (flags & MDB_CURRENT) {
 		if (!(mc->mc_flags & C_INITIALIZED))
 			return EINVAL;
 		rc = MDB_SUCCESS;
@@ -8321,7 +8321,7 @@ put_sub:
 			xdata.mv_size = 0;
 			xdata.mv_data = "";
 			leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
-			if (flags & MDB_CURRENT) {
+			if (flags == MDB_CURRENT) {
 				xflags = MDB_CURRENT|MDB_NOSPILL;
 			} else {
 				mdb_xcursor_init1(mc, leaf);
diff --git a/libraries/liblmdb/mdb_load.1 b/libraries/liblmdb/mdb_load.1
index e25ff5a2b7..f897208846 100644
--- a/libraries/liblmdb/mdb_load.1
+++ b/libraries/liblmdb/mdb_load.1
@@ -37,6 +37,13 @@ option below.
 .BR \-V
 Write the library version number to the standard output, and exit.
 .TP
+.BR \-a
+Append all records in the order they appear in the input. The input is assumed to already be
+in correctly sorted order and no sorting or checking for redundant values will be performed.
+This option must be used to reload data that was produced by running
+.B mdb_dump
+on a database that uses custom compare functions.
+.TP
 .BR \-f \ file
 Read from the specified file instead of from the standard input.
 .TP
diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c
index 7888144a7e..3865e5a290 100644
--- a/libraries/liblmdb/mdb_load.c
+++ b/libraries/liblmdb/mdb_load.c
@@ -37,6 +37,7 @@ static int Eof;
 static MDB_envinfo info;
 
 static MDB_val kbuf, dbuf;
+static MDB_val k0buf;
 
 #define Yu	MDB_PRIy(u)
 
@@ -274,10 +275,15 @@ badend:
 
 static void usage(void)
 {
-	fprintf(stderr, "usage: %s [-V] [-f input] [-n] [-s name] [-N] [-T] dbpath\n", prog);
+	fprintf(stderr, "usage: %s [-V] [-a] [-f input] [-n] [-s name] [-N] [-T] dbpath\n", prog);
 	exit(EXIT_FAILURE);
 }
 
+static int greater(const MDB_val *a, const MDB_val *b)
+{
+	return 1;
+}
+
 int main(int argc, char *argv[])
 {
 	int i, rc;
@@ -286,8 +292,9 @@ int main(int argc, char *argv[])
 	MDB_cursor *mc;
 	MDB_dbi dbi;
 	char *envname;
-	int envflags = 0, putflags = 0;
-	int dohdr = 0;
+	int envflags = MDB_NOSYNC, putflags = 0;
+	int dohdr = 0, append = 0;
+	MDB_val prevk;
 
 	prog = argv[0];
 
@@ -295,19 +302,23 @@ int main(int argc, char *argv[])
 		usage();
 	}
 
-	/* -f: load file instead of stdin
+	/* -a: append records in input order
+	 * -f: load file instead of stdin
 	 * -n: use NOSUBDIR flag on env_open
 	 * -s: load into named subDB
 	 * -N: use NOOVERWRITE on puts
 	 * -T: read plaintext
 	 * -V: print version and exit
 	 */
-	while ((i = getopt(argc, argv, "f:ns:NTV")) != EOF) {
+	while ((i = getopt(argc, argv, "af:ns:NTV")) != EOF) {
 		switch(i) {
 		case 'V':
 			printf("%s\n", MDB_VERSION_STRING);
 			exit(0);
 			break;
+		case 'a':
+			append = 1;
+			break;
 		case 'f':
 			if (freopen(optarg, "r", stdin) == NULL) {
 				fprintf(stderr, "%s: %s: reopen: %s\n",
@@ -366,12 +377,16 @@ int main(int argc, char *argv[])
 	}
 
 	kbuf.mv_size = mdb_env_get_maxkeysize(env) * 2 + 2;
-	kbuf.mv_data = malloc(kbuf.mv_size);
+	kbuf.mv_data = malloc(kbuf.mv_size * 2);
+	k0buf.mv_size = kbuf.mv_size;
+	k0buf.mv_data = (char *)kbuf.mv_data + kbuf.mv_size;
+	prevk.mv_data = k0buf.mv_data;
 
 	while(!Eof) {
 		MDB_val key, data;
 		int batch = 0;
 		flags = 0;
+		int appflag;
 
 		if (!dohdr) {
 			dohdr = 1;
@@ -389,6 +404,12 @@ int main(int argc, char *argv[])
 			fprintf(stderr, "mdb_dbi_open failed, error %d %s\n", rc, mdb_strerror(rc));
 			goto txn_abort;
 		}
+		prevk.mv_size = 0;
+		if (append) {
+			mdb_set_compare(txn, dbi, greater);
+			if (flags & MDB_DUPSORT)
+				mdb_set_dupsort(txn, dbi, greater);
+		}
 
 		rc = mdb_cursor_open(txn, dbi, &mc);
 		if (rc) {
@@ -407,7 +428,20 @@ int main(int argc, char *argv[])
 				goto txn_abort;
 			}
 
-			rc = mdb_cursor_put(mc, &key, &data, putflags);
+			if (append) {
+				appflag = MDB_APPEND;
+				if (flags & MDB_DUPSORT) {
+					if (prevk.mv_size == key.mv_size && !memcmp(prevk.mv_data, key.mv_data, key.mv_size))
+						appflag = MDB_CURRENT|MDB_APPENDDUP;
+					else {
+						memcpy(prevk.mv_data, key.mv_data, key.mv_size);
+						prevk.mv_size = key.mv_size;
+					}
+				}
+			} else {
+				appflag = 0;
+			}
+			rc = mdb_cursor_put(mc, &key, &data, putflags|appflag);
 			if (rc == MDB_KEYEXIST && putflags)
 				continue;
 			if (rc) {
@@ -432,6 +466,10 @@ int main(int argc, char *argv[])
 					fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
 					goto txn_abort;
 				}
+				if (appflag & MDB_APPENDDUP) {
+					MDB_val k, d;
+					mdb_cursor_get(mc, &k, &d, MDB_LAST);
+				}
 				batch = 0;
 			}
 		}

From 4d747ff2f94ca4d8a8985ef777960e3fe0244b47 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 26 Oct 2017 19:04:37 +0100
Subject: [PATCH 357/504] ITS#8760 fix regression in 0.9.19

---
 libraries/liblmdb/mdb.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index f3aa3c07ad..2a04196921 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -11188,8 +11188,11 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db
 		MDB_node *node = NODEPTR(mc.mc_pg[mc.mc_top], mc.mc_ki[mc.mc_top]);
 		if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) != F_SUBDATA)
 			return MDB_INCOMPATIBLE;
-	} else if (! (rc == MDB_NOTFOUND && (flags & MDB_CREATE))) {
-		return rc;
+	} else {
+		if (rc != MDB_NOTFOUND || !(flags & MDB_CREATE))
+			return rc;
+		if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY))
+			return EACCES;
 	}
 
 	/* Done here so we cannot fail after creating a new DB */

From ec3e4ed9d80b42743d5e87e1eba553c500334fc5 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sun, 19 Nov 2017 19:44:53 +0000
Subject: [PATCH 358/504] Add mdb_drop tool

---
 libraries/liblmdb/Makefile   |   5 +-
 libraries/liblmdb/mdb_drop.1 |  40 +++++++++++
 libraries/liblmdb/mdb_drop.c | 135 +++++++++++++++++++++++++++++++++++
 libraries/liblmdb/tooltag    |   5 ++
 4 files changed, 183 insertions(+), 2 deletions(-)
 create mode 100644 libraries/liblmdb/mdb_drop.1
 create mode 100644 libraries/liblmdb/mdb_drop.c

diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile
index 2ff32345ce..859379e1bd 100644
--- a/libraries/liblmdb/Makefile
+++ b/libraries/liblmdb/Makefile
@@ -39,8 +39,8 @@ mandir = $(datarootdir)/man
 
 IHDRS	= lmdb.h
 ILIBS	= liblmdb.a liblmdb$(SOEXT)
-IPROGS	= mdb_stat mdb_copy mdb_dump mdb_load
-IDOCS	= mdb_stat.1 mdb_copy.1 mdb_dump.1 mdb_load.1
+IPROGS	= mdb_stat mdb_copy mdb_dump mdb_load mdb_drop
+IDOCS	= mdb_stat.1 mdb_copy.1 mdb_dump.1 mdb_load.1 mdb_drop.1
 PROGS	= $(IPROGS) mtest mtest2 mtest3 mtest4 mtest5
 RPROGS	= mtest_remap mtest_enc
 
@@ -76,6 +76,7 @@ mdb_stat: mdb_stat.o liblmdb.a
 mdb_copy: mdb_copy.o liblmdb.a
 mdb_dump: mdb_dump.o liblmdb.a
 mdb_load: mdb_load.o liblmdb.a
+mdb_drop: mdb_drop.o liblmdb.a
 mtest:    mtest.o    liblmdb.a
 mtest2:	mtest2.o liblmdb.a
 mtest3:	mtest3.o liblmdb.a
diff --git a/libraries/liblmdb/mdb_drop.1 b/libraries/liblmdb/mdb_drop.1
new file mode 100644
index 0000000000..997e291182
--- /dev/null
+++ b/libraries/liblmdb/mdb_drop.1
@@ -0,0 +1,40 @@
+.TH MDB_DROP 1 "2017/11/19" "LMDB 0.9.70"
+.\" Copyright 2014-2017 Howard Chu, Symas Corp. All Rights Reserved.
+.\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
+.SH NAME
+mdb_drop \- LMDB database delete tool
+.SH SYNOPSIS
+.B mdb_drop
+[\c
+.BR \-V ]
+[\c
+.BR \-n ]
+[\c
+.BR \-d ]
+[\c
+.BI \-s \ subdb\fR]
+.BR \ envpath
+.SH DESCRIPTION
+The
+.B mdb_drop
+utility empties or deletes a database in the specified
+environment.
+.SH OPTIONS
+.TP
+.BR \-V
+Write the library version number to the standard output, and exit.
+.TP
+.BR \-n
+Operate on an LMDB database which does not use subdirectories.
+.TP
+.BR \-d
+Delete the specified database, don't just empty it.
+.TP
+.BR \-s \ subdb
+Operate on a specific subdatabase. If no database is specified, only the main database is dropped.
+.SH DIAGNOSTICS
+Exit status is zero if no errors occur.
+Errors result in a non-zero exit status and
+a diagnostic message being written to standard error.
+.SH AUTHOR
+Howard Chu of Symas Corporation <http://www.symas.com>
diff --git a/libraries/liblmdb/mdb_drop.c b/libraries/liblmdb/mdb_drop.c
new file mode 100644
index 0000000000..725891685a
--- /dev/null
+++ b/libraries/liblmdb/mdb_drop.c
@@ -0,0 +1,135 @@
+/* mdb_drop.c - memory-mapped database delete tool */
+/*
+ * Copyright 2016-2017 Howard Chu, Symas Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
+ */
+#include <stdio.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#include <unistd.h>
+#include <signal.h>
+#include "lmdb.h"
+
+static volatile sig_atomic_t gotsig;
+
+static void dumpsig( int sig )
+{
+	gotsig=1;
+}
+
+static void usage(char *prog)
+{
+	fprintf(stderr, "usage: %s [-V] [-n] [-d] [-s subdb] dbpath\n", prog);
+	exit(EXIT_FAILURE);
+}
+
+int main(int argc, char *argv[])
+{
+	int i, rc;
+	MDB_env *env;
+	MDB_txn *txn;
+	MDB_dbi dbi;
+	char *prog = argv[0];
+	char *envname;
+	char *subname = NULL;
+	int envflags = 0, delete = 0;
+
+	if (argc < 2) {
+		usage(prog);
+	}
+
+	/* -d: delete the db, don't just empty it
+	 * -s: drop the named subDB
+	 * -n: use NOSUBDIR flag on env_open
+	 * -V: print version and exit
+	 * (default) empty the main DB
+	 */
+	while ((i = getopt(argc, argv, "dns:V")) != EOF) {
+		switch(i) {
+		case 'V':
+			printf("%s\n", MDB_VERSION_STRING);
+			exit(0);
+			break;
+		case 'd':
+			delete = 1;
+			break;
+		case 'n':
+			envflags |= MDB_NOSUBDIR;
+			break;
+		case 's':
+			subname = optarg;
+			break;
+		default:
+			usage(prog);
+		}
+	}
+
+	if (optind != argc - 1)
+		usage(prog);
+
+#ifdef SIGPIPE
+	signal(SIGPIPE, dumpsig);
+#endif
+#ifdef SIGHUP
+	signal(SIGHUP, dumpsig);
+#endif
+	signal(SIGINT, dumpsig);
+	signal(SIGTERM, dumpsig);
+
+	envname = argv[optind];
+	rc = mdb_env_create(&env);
+	if (rc) {
+		fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc));
+		return EXIT_FAILURE;
+	}
+
+	mdb_env_set_maxdbs(env, 2);
+
+	rc = mdb_env_open(env, envname, envflags, 0664);
+	if (rc) {
+		fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
+		goto env_close;
+	}
+
+	rc = mdb_txn_begin(env, NULL, 0, &txn);
+	if (rc) {
+		fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
+		goto env_close;
+	}
+
+	rc = mdb_open(txn, subname, 0, &dbi);
+	if (rc) {
+		fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
+		goto txn_abort;
+	}
+
+	rc = mdb_drop(txn, dbi, delete);
+	if (rc) {
+		fprintf(stderr, "mdb_drop failed, error %d %s\n", rc, mdb_strerror(rc));
+		goto txn_abort;
+	}
+	rc = mdb_txn_commit(txn);
+	if (rc) {
+		fprintf(stderr, "mdb_txn_commit failed, error %d %s\n", rc, mdb_strerror(rc));
+		goto txn_abort;
+	}
+	txn = NULL;
+
+txn_abort:
+	if (txn)
+		mdb_txn_abort(txn);
+env_close:
+	mdb_env_close(env);
+
+	return rc ? EXIT_FAILURE : EXIT_SUCCESS;
+}
diff --git a/libraries/liblmdb/tooltag b/libraries/liblmdb/tooltag
index 229bf16baa..879c14d227 100644
--- a/libraries/liblmdb/tooltag
+++ b/libraries/liblmdb/tooltag
@@ -4,6 +4,11 @@
 	<title>mdb_copy - environment copy tool</title>
 	<filename>mdb_copy.1</filename>
   </compound>
+  <compound kind="page">
+    <name>mdb_drop_1</name>
+	<title>mdb_drop - database delete tool</title>
+	<filename>mdb_drop.1</filename>
+  </compound>
   <compound kind="page">
     <name>mdb_dump_1</name>
 	<title>mdb_dump - environment export tool</title>

From bdfb167105ef963da4deff1c3c8bc3a0b3cb4053 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sun, 18 Feb 2018 21:00:54 +0000
Subject: [PATCH 359/504] ITS#8324 More for Win32 NTDLL junk

Use GetProcAddress at runtime, avoid buildtime NTDLL link issues
---
 libraries/liblmdb/Makefile |  4 ++--
 libraries/liblmdb/mdb.c    | 37 +++++++++++++++++++++++++++++--------
 2 files changed, 31 insertions(+), 10 deletions(-)

diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile
index 859379e1bd..292b0fa592 100644
--- a/libraries/liblmdb/Makefile
+++ b/libraries/liblmdb/Makefile
@@ -24,8 +24,8 @@ W	= -W -Wall -Wno-unused-parameter -Wbad-function-cast -Wuninitialized
 THREADS = -pthread
 OPT = -O2 -g
 CFLAGS	= $(THREADS) $(OPT) $(W) $(XCFLAGS)
-LDLIBS	= # -lntdll # Windows needs ntdll
-SOLIBS	= # -lntdll
+LDLIBS	= 
+SOLIBS	= 
 SOEXT	= .so
 prefix	= /usr/local
 exec_prefix = $(prefix)
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 2a04196921..fd7c76f2e0 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -48,29 +48,35 @@
  * the full size. These APIs are defined in <wdm.h> and <ntifs.h>
  * but those headers are meant for driver-level development and
  * conflict with the regular user-level headers, so we explicitly
- * declare them here. Using these APIs also means we must link to
- * ntdll.dll, which is not linked by default in user code.
+ * declare them here. We get pointers to these functions from
+ * NTDLL.DLL at runtime, to avoid buildtime dependencies on any
+ * NTDLL import libraries.
  */
-NTSTATUS WINAPI
-NtCreateSection(OUT PHANDLE sh, IN ACCESS_MASK acc,
+typedef NTSTATUS WINAPI (NtCreateSectionFunc)
+  (OUT PHANDLE sh, IN ACCESS_MASK acc,
   IN void * oa OPTIONAL,
   IN PLARGE_INTEGER ms OPTIONAL,
   IN ULONG pp, IN ULONG aa, IN HANDLE fh OPTIONAL);
 
+static NtCreateSectionFunc *NtCreateSection;
+
 typedef enum _SECTION_INHERIT {
 	ViewShare = 1,
 	ViewUnmap = 2
 } SECTION_INHERIT;
 
-NTSTATUS WINAPI
-NtMapViewOfSection(IN PHANDLE sh, IN HANDLE ph,
+typedef NTSTATUS WINAPI (NtMapViewOfSectionFunc)
+  (IN PHANDLE sh, IN HANDLE ph,
   IN OUT PVOID *addr, IN ULONG_PTR zbits,
   IN SIZE_T cs, IN OUT PLARGE_INTEGER off OPTIONAL,
   IN OUT PSIZE_T vs, IN SECTION_INHERIT ih,
   IN ULONG at, IN ULONG pp);
 
-NTSTATUS WINAPI
-NtClose(HANDLE h);
+static NtMapViewOfSectionFunc *NtMapViewOfSection;
+
+typedef NTSTATUS WINAPI (NtCloseFunc)(HANDLE h);
+
+static NtCloseFunc *NtClose;
 
 /** getpid() returns int; MinGW defines pid_t but MinGW64 typedefs it
  *  as int64 which is wrong. MSVC doesn't define it at all, so just
@@ -5014,6 +5020,21 @@ mdb_env_open2(MDB_env *env, int prev)
 		env->me_pidquery = MDB_PROCESS_QUERY_LIMITED_INFORMATION;
 	else
 		env->me_pidquery = PROCESS_QUERY_INFORMATION;
+	/* Grab functions we need from NTDLL */
+	if (!NtCreateSection) {
+		HMODULE h = GetModuleHandle("NTDLL.DLL");
+		if (!h)
+			return MDB_PROBLEM;
+		NtClose = (NtCloseFunc *)GetProcAddress(h, "NtClose");
+		if (!NtClose)
+			return MDB_PROBLEM;
+		NtMapViewOfSection = (NtMapViewOfSectionFunc *)GetProcAddress(h, "NtMapViewOfSection");
+		if (!NtMapViewOfSection)
+			return MDB_PROBLEM;
+		NtCreateSection = (NtCreateSectionFunc *)GetProcAddress(h, "NtCreateSection");
+		if (!NtCreateSection)
+			return MDB_PROBLEM;
+	}
 #endif /* _WIN32 */
 
 #ifdef BROKEN_FDATASYNC

From d52328b75d08dfa8acd6d72f9fcc777ec90e7dbf Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 20 Mar 2018 18:34:56 +0000
Subject: [PATCH 360/504] ITS#8819 can't use fakepage mp_ptrs directly

---
 libraries/liblmdb/mdb.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index fd7c76f2e0..272ac76d2f 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -8198,8 +8198,9 @@ prep_subDB:
 				} else {
 					memcpy((char *)mp + mp->mp_upper + PAGEBASE, (char *)fp + fp->mp_upper + PAGEBASE,
 						olddata.mv_size - fp->mp_upper - PAGEBASE);
+					memcpy((char *)(&mp->mp_ptrs), (char *)(&fp->mp_ptrs), NUMKEYS(fp) * sizeof(mp->mp_ptrs[0]));
 					for (i=0; i<NUMKEYS(fp); i++)
-						mp->mp_ptrs[i] = fp->mp_ptrs[i] + offset;
+						mp->mp_ptrs[i] += offset;
 				}
 			}
 

From 5a5e056f2256554ab3b4fcd58fdd2bde45f585cf Mon Sep 17 00:00:00 2001
From: Quanah Gibson-Mount <quanah@openldap.org>
Date: Thu, 22 Mar 2018 15:20:57 +0000
Subject: [PATCH 361/504] Happy New Year

---
 libraries/liblmdb/COPYRIGHT      | 2 +-
 libraries/liblmdb/intro.doc      | 2 +-
 libraries/liblmdb/lmdb.h         | 2 +-
 libraries/liblmdb/mdb.c          | 2 +-
 libraries/liblmdb/mdb_copy.1     | 2 +-
 libraries/liblmdb/mdb_copy.c     | 2 +-
 libraries/liblmdb/mdb_drop.1     | 4 ++--
 libraries/liblmdb/mdb_drop.c     | 2 +-
 libraries/liblmdb/mdb_dump.1     | 2 +-
 libraries/liblmdb/mdb_dump.c     | 2 +-
 libraries/liblmdb/mdb_load.1     | 4 ++--
 libraries/liblmdb/mdb_load.c     | 2 +-
 libraries/liblmdb/mdb_stat.1     | 2 +-
 libraries/liblmdb/mdb_stat.c     | 2 +-
 libraries/liblmdb/midl.c         | 4 ++--
 libraries/liblmdb/midl.h         | 4 ++--
 libraries/liblmdb/mtest.c        | 2 +-
 libraries/liblmdb/mtest2.c       | 2 +-
 libraries/liblmdb/mtest3.c       | 2 +-
 libraries/liblmdb/mtest4.c       | 2 +-
 libraries/liblmdb/mtest5.c       | 2 +-
 libraries/liblmdb/mtest6.c       | 2 +-
 libraries/liblmdb/sample-bdb.txt | 2 +-
 libraries/liblmdb/sample-mdb.txt | 2 +-
 24 files changed, 28 insertions(+), 28 deletions(-)

diff --git a/libraries/liblmdb/COPYRIGHT b/libraries/liblmdb/COPYRIGHT
index 1508a735b6..10d7b3204f 100644
--- a/libraries/liblmdb/COPYRIGHT
+++ b/libraries/liblmdb/COPYRIGHT
@@ -1,4 +1,4 @@
-Copyright 2011-2017 Howard Chu, Symas Corp.
+Copyright 2011-2018 Howard Chu, Symas Corp.
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/intro.doc b/libraries/liblmdb/intro.doc
index f7bd8c0d02..64dfcaad84 100644
--- a/libraries/liblmdb/intro.doc
+++ b/libraries/liblmdb/intro.doc
@@ -1,5 +1,5 @@
 /*
- * Copyright 2015-2017 Howard Chu, Symas Corp.
+ * Copyright 2015-2018 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index bc1e2684ba..383d9c8bac 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -136,7 +136,7 @@
  *
  *	@author	Howard Chu, Symas Corporation.
  *
- *	@copyright Copyright 2011-2017 Howard Chu, Symas Corp. All rights reserved.
+ *	@copyright Copyright 2011-2018 Howard Chu, Symas Corp. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted only as authorized by the OpenLDAP
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 272ac76d2f..9782de363d 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5,7 +5,7 @@
  *	BerkeleyDB API, but much simplified.
  */
 /*
- * Copyright 2011-2017 Howard Chu, Symas Corp.
+ * Copyright 2011-2018 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mdb_copy.1 b/libraries/liblmdb/mdb_copy.1
index edcb9a1659..d5a95014a2 100644
--- a/libraries/liblmdb/mdb_copy.1
+++ b/libraries/liblmdb/mdb_copy.1
@@ -1,5 +1,5 @@
 .TH MDB_COPY 1 "2017/07/31" "LMDB 0.9.90"
-.\" Copyright 2012-2017 Howard Chu, Symas Corp. All Rights Reserved.
+.\" Copyright 2012-2018 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_copy \- LMDB environment copy tool
diff --git a/libraries/liblmdb/mdb_copy.c b/libraries/liblmdb/mdb_copy.c
index fd7268dc4d..23be815062 100644
--- a/libraries/liblmdb/mdb_copy.c
+++ b/libraries/liblmdb/mdb_copy.c
@@ -1,6 +1,6 @@
 /* mdb_copy.c - memory-mapped database backup tool */
 /*
- * Copyright 2012-2017 Howard Chu, Symas Corp.
+ * Copyright 2012-2018 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mdb_drop.1 b/libraries/liblmdb/mdb_drop.1
index 997e291182..370d89c032 100644
--- a/libraries/liblmdb/mdb_drop.1
+++ b/libraries/liblmdb/mdb_drop.1
@@ -1,5 +1,5 @@
-.TH MDB_DROP 1 "2017/11/19" "LMDB 0.9.70"
-.\" Copyright 2014-2017 Howard Chu, Symas Corp. All Rights Reserved.
+.TH MDB_DROP 1 "2017/11/19" "LMDB 0.9.90"
+.\" Copyright 2014-2018 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_drop \- LMDB database delete tool
diff --git a/libraries/liblmdb/mdb_drop.c b/libraries/liblmdb/mdb_drop.c
index 725891685a..7d2af4b62e 100644
--- a/libraries/liblmdb/mdb_drop.c
+++ b/libraries/liblmdb/mdb_drop.c
@@ -1,6 +1,6 @@
 /* mdb_drop.c - memory-mapped database delete tool */
 /*
- * Copyright 2016-2017 Howard Chu, Symas Corp.
+ * Copyright 2016-2018 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mdb_dump.1 b/libraries/liblmdb/mdb_dump.1
index 37ccae42f2..65aafbacdc 100644
--- a/libraries/liblmdb/mdb_dump.1
+++ b/libraries/liblmdb/mdb_dump.1
@@ -1,5 +1,5 @@
 .TH MDB_DUMP 1 "2017/07/31" "LMDB 0.9.90"
-.\" Copyright 2014-2017 Howard Chu, Symas Corp. All Rights Reserved.
+.\" Copyright 2014-2018 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_dump \- LMDB environment export tool
diff --git a/libraries/liblmdb/mdb_dump.c b/libraries/liblmdb/mdb_dump.c
index fa1ca2966b..87619bf49a 100644
--- a/libraries/liblmdb/mdb_dump.c
+++ b/libraries/liblmdb/mdb_dump.c
@@ -1,6 +1,6 @@
 /* mdb_dump.c - memory-mapped database dump tool */
 /*
- * Copyright 2011-2017 Howard Chu, Symas Corp.
+ * Copyright 2011-2018 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mdb_load.1 b/libraries/liblmdb/mdb_load.1
index f897208846..7bfaa9126c 100644
--- a/libraries/liblmdb/mdb_load.1
+++ b/libraries/liblmdb/mdb_load.1
@@ -1,5 +1,5 @@
-.TH MDB_LOAD 1 "2015/09/30" "LMDB 0.9.17"
-.\" Copyright 2014-2017 Howard Chu, Symas Corp. All Rights Reserved.
+.TH MDB_LOAD 1 "2015/09/30" "LMDB 0.9.90"
+.\" Copyright 2014-2018 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_load \- LMDB environment import tool
diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c
index 3865e5a290..7275d7a4f3 100644
--- a/libraries/liblmdb/mdb_load.c
+++ b/libraries/liblmdb/mdb_load.c
@@ -1,6 +1,6 @@
 /* mdb_load.c - memory-mapped database load tool */
 /*
- * Copyright 2011-2017 Howard Chu, Symas Corp.
+ * Copyright 2011-2018 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mdb_stat.1 b/libraries/liblmdb/mdb_stat.1
index 236019dc6e..f5dd86e228 100644
--- a/libraries/liblmdb/mdb_stat.1
+++ b/libraries/liblmdb/mdb_stat.1
@@ -1,5 +1,5 @@
 .TH MDB_STAT 1 "2017/07/31" "LMDB 0.9.90"
-.\" Copyright 2012-2017 Howard Chu, Symas Corp. All Rights Reserved.
+.\" Copyright 2012-2018 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_stat \- LMDB environment status tool
diff --git a/libraries/liblmdb/mdb_stat.c b/libraries/liblmdb/mdb_stat.c
index f274a4666c..a7f7eff450 100644
--- a/libraries/liblmdb/mdb_stat.c
+++ b/libraries/liblmdb/mdb_stat.c
@@ -1,6 +1,6 @@
 /* mdb_stat.c - memory-mapped database status tool */
 /*
- * Copyright 2011-2017 Howard Chu, Symas Corp.
+ * Copyright 2011-2018 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/midl.c b/libraries/liblmdb/midl.c
index c5e5828a86..928dcca1cc 100644
--- a/libraries/liblmdb/midl.c
+++ b/libraries/liblmdb/midl.c
@@ -3,8 +3,8 @@
 /* $OpenLDAP$ */
 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
  *
- * Copyright 2000-2016 The OpenLDAP Foundation.
- * Portions Copyright 2001-2017 Howard Chu, Symas Corp.
+ * Copyright 2000-2018 The OpenLDAP Foundation.
+ * Portions Copyright 2001-2018 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/midl.h b/libraries/liblmdb/midl.h
index c814d45583..474d521aeb 100644
--- a/libraries/liblmdb/midl.h
+++ b/libraries/liblmdb/midl.h
@@ -11,8 +11,8 @@
 /* $OpenLDAP$ */
 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
  *
- * Copyright 2000-2016 The OpenLDAP Foundation.
- * Portions Copyright 2001-2017 Howard Chu, Symas Corp.
+ * Copyright 2000-2018 The OpenLDAP Foundation.
+ * Portions Copyright 2001-2018 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest.c b/libraries/liblmdb/mtest.c
index 4a2eafd033..6fc5840c3f 100644
--- a/libraries/liblmdb/mtest.c
+++ b/libraries/liblmdb/mtest.c
@@ -1,6 +1,6 @@
 /* mtest.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2017 Howard Chu, Symas Corp.
+ * Copyright 2011-2018 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest2.c b/libraries/liblmdb/mtest2.c
index 9691c04fad..64b742aa46 100644
--- a/libraries/liblmdb/mtest2.c
+++ b/libraries/liblmdb/mtest2.c
@@ -1,6 +1,6 @@
 /* mtest2.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2017 Howard Chu, Symas Corp.
+ * Copyright 2011-2018 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest3.c b/libraries/liblmdb/mtest3.c
index 4390d7ab4a..81e4bbf9b0 100644
--- a/libraries/liblmdb/mtest3.c
+++ b/libraries/liblmdb/mtest3.c
@@ -1,6 +1,6 @@
 /* mtest3.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2017 Howard Chu, Symas Corp.
+ * Copyright 2011-2018 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest4.c b/libraries/liblmdb/mtest4.c
index 9f7884a93f..c355cf105a 100644
--- a/libraries/liblmdb/mtest4.c
+++ b/libraries/liblmdb/mtest4.c
@@ -1,6 +1,6 @@
 /* mtest4.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2017 Howard Chu, Symas Corp.
+ * Copyright 2011-2018 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest5.c b/libraries/liblmdb/mtest5.c
index 4d34247e60..95793ec1fb 100644
--- a/libraries/liblmdb/mtest5.c
+++ b/libraries/liblmdb/mtest5.c
@@ -1,6 +1,6 @@
 /* mtest5.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2017 Howard Chu, Symas Corp.
+ * Copyright 2011-2018 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest6.c b/libraries/liblmdb/mtest6.c
index 26904e88ba..cb0d4d73c9 100644
--- a/libraries/liblmdb/mtest6.c
+++ b/libraries/liblmdb/mtest6.c
@@ -1,6 +1,6 @@
 /* mtest6.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2017 Howard Chu, Symas Corp.
+ * Copyright 2011-2018 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/sample-bdb.txt b/libraries/liblmdb/sample-bdb.txt
index 4de26d5585..97220f0ed2 100644
--- a/libraries/liblmdb/sample-bdb.txt
+++ b/libraries/liblmdb/sample-bdb.txt
@@ -3,7 +3,7 @@
  * Do a line-by-line comparison of this and sample-mdb.txt
  */
 /*
- * Copyright 2012-2017 Howard Chu, Symas Corp.
+ * Copyright 2012-2018 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/sample-mdb.txt b/libraries/liblmdb/sample-mdb.txt
index 6e3622b866..1d20ed3d02 100644
--- a/libraries/liblmdb/sample-mdb.txt
+++ b/libraries/liblmdb/sample-mdb.txt
@@ -3,7 +3,7 @@
  * Do a line-by-line comparison of this and sample-bdb.txt
  */
 /*
- * Copyright 2012-2017 Howard Chu, Symas Corp.
+ * Copyright 2012-2018 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without

From 8218d067177a6a4fb9ecb92cb4bdc730c3e079c1 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 2 Apr 2018 18:01:19 +0100
Subject: [PATCH 362/504] ITS#8831 move flag init into readhdr

Avoid stomping on flags from 1st readhr invocation
---
 libraries/liblmdb/mdb_load.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c
index 7275d7a4f3..d313813e31 100644
--- a/libraries/liblmdb/mdb_load.c
+++ b/libraries/liblmdb/mdb_load.c
@@ -65,6 +65,7 @@ static void readhdr(void)
 {
 	char *ptr;
 
+	flags = 0;
 	while (fgets(dbuf.mv_data, dbuf.mv_size, stdin) != NULL) {
 		lineno++;
 		if (!strncmp(dbuf.mv_data, "VERSION=", STRLENOF("VERSION="))) {
@@ -385,7 +386,6 @@ int main(int argc, char *argv[])
 	while(!Eof) {
 		MDB_val key, data;
 		int batch = 0;
-		flags = 0;
 		int appflag;
 
 		if (!dohdr) {

From bfe439cd52ff49c5052cbcc31de9c2b748ab18af Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 2 May 2018 17:05:29 +0100
Subject: [PATCH 363/504] ITS#8844 use getpid() in mdb_env_close0()

---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 9782de363d..be4161013a 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5997,7 +5997,7 @@ mdb_env_close_active(MDB_env *env, int excl)
 	if (env->me_fd != INVALID_HANDLE_VALUE)
 		(void) close(env->me_fd);
 	if (env->me_txns) {
-		MDB_PID_T pid = env->me_pid;
+		MDB_PID_T pid = getpid();
 		/* Clearing readers is done in this function because
 		 * me_txkey with its destructor must be disabled first.
 		 *

From b65765443a810caacbb843ea974a5683a261f2dc Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Fri, 22 Jun 2018 16:30:13 +0100
Subject: [PATCH 364/504] ITS#8756 remove loose pg from dirty list in
 freelist_save

---
 libraries/liblmdb/mdb.c | 33 ++++++++++++++++++++++++++++++++-
 1 file changed, 32 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index be4161013a..671a86e497 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -3677,11 +3677,42 @@ mdb_freelist_save(MDB_txn *txn)
 		 * we may be unable to return them to me_pghead.
 		 */
 		MDB_page *mp = txn->mt_loose_pgs;
+		MDB_ID2 *dl = txn->mt_u.dirty_list;
+		unsigned x;
 		if ((rc = mdb_midl_need(&txn->mt_free_pgs, txn->mt_loose_count)) != 0)
 			return rc;
 		lost_loose = txn->mt_loose_count;
-		for (; mp; mp = NEXT_LOOSE_PAGE(mp))
+		for (; mp; mp = NEXT_LOOSE_PAGE(mp)) {
 			mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno);
+			/* must also remove from dirty list */
+			if (txn->mt_flags & MDB_TXN_WRITEMAP) {
+				for (x=1; x<=dl[0].mid; x++)
+					if (dl[x].mid == mp->mp_pgno)
+						break;
+				mdb_tassert(txn, x <= dl[0].mid);
+			} else {
+				x = mdb_mid2l_search(dl, mp->mp_pgno);
+				mdb_tassert(txn, dl[x].mid == mp->mp_pgno);
+			}
+			dl[x].mptr = NULL;
+			mdb_dpage_free(env, mp);
+		}
+		{
+			/* squash freed slots out of the dirty list */
+			unsigned y;
+			for (y=1; dl[y].mptr && y <= dl[0].mid; y++);
+			if (y <= dl[0].mid) {
+				for(x=y, y++;;) {
+					while (!dl[y].mptr && y <= dl[0].mid) y++;
+					if (y > dl[0].mid) break;
+					dl[x++] = dl[y++];
+				}
+				dl[0].mid = x-1;
+			} else {
+				/* all slots freed */
+				dl[0].mid = 0;
+			}
+		}
 		txn->mt_loose_pgs = NULL;
 		txn->mt_loose_count = 0;
 	}

From 279cc7b83696c082265fad4cc9ce1f6a03e662d0 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 6 Aug 2018 13:10:59 +0100
Subject: [PATCH 365/504] ITS#8891 fix M$ WINAPI typedefs

---
 libraries/liblmdb/mdb.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 671a86e497..ad4a3fcb01 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -52,7 +52,7 @@
  * NTDLL.DLL at runtime, to avoid buildtime dependencies on any
  * NTDLL import libraries.
  */
-typedef NTSTATUS WINAPI (NtCreateSectionFunc)
+typedef NTSTATUS (WINAPI NtCreateSectionFunc)
   (OUT PHANDLE sh, IN ACCESS_MASK acc,
   IN void * oa OPTIONAL,
   IN PLARGE_INTEGER ms OPTIONAL,
@@ -65,7 +65,7 @@ typedef enum _SECTION_INHERIT {
 	ViewUnmap = 2
 } SECTION_INHERIT;
 
-typedef NTSTATUS WINAPI (NtMapViewOfSectionFunc)
+typedef NTSTATUS (WINAPI NtMapViewOfSectionFunc)
   (IN PHANDLE sh, IN HANDLE ph,
   IN OUT PVOID *addr, IN ULONG_PTR zbits,
   IN SIZE_T cs, IN OUT PLARGE_INTEGER off OPTIONAL,
@@ -74,7 +74,7 @@ typedef NTSTATUS WINAPI (NtMapViewOfSectionFunc)
 
 static NtMapViewOfSectionFunc *NtMapViewOfSection;
 
-typedef NTSTATUS WINAPI (NtCloseFunc)(HANDLE h);
+typedef NTSTATUS (WINAPI NtCloseFunc)(HANDLE h);
 
 static NtCloseFunc *NtClose;
 

From 0612992edf3de0df52c8d5ba81cef9e2f88996e9 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 29 Aug 2018 01:25:01 +0100
Subject: [PATCH 366/504] ITS#8908 DOC: GET_MULTIPLE etc don't return the key

Unnecessary since these are DUPs, the key will always be the same
---
 libraries/liblmdb/lmdb.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 383d9c8bac..f427ea58be 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -426,7 +426,7 @@ typedef enum MDB_cursor_op {
 	MDB_GET_BOTH,			/**< Position at key/data pair. Only for #MDB_DUPSORT */
 	MDB_GET_BOTH_RANGE,		/**< position at key, nearest data. Only for #MDB_DUPSORT */
 	MDB_GET_CURRENT,		/**< Return key/data at current cursor position */
-	MDB_GET_MULTIPLE,		/**< Return key and up to a page of duplicate data items
+	MDB_GET_MULTIPLE,		/**< Return up to a page of duplicate data items
 								from current cursor position. Move cursor to prepare
 								for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */
 	MDB_LAST,				/**< Position at last key/data item */
@@ -435,7 +435,7 @@ typedef enum MDB_cursor_op {
 	MDB_NEXT,				/**< Position at next data item */
 	MDB_NEXT_DUP,			/**< Position at next data item of current key.
 								Only for #MDB_DUPSORT */
-	MDB_NEXT_MULTIPLE,		/**< Return key and up to a page of duplicate data items
+	MDB_NEXT_MULTIPLE,		/**< Return up to a page of duplicate data items
 								from next cursor position. Move cursor to prepare
 								for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */
 	MDB_NEXT_NODUP,			/**< Position at first data item of next key */
@@ -446,7 +446,7 @@ typedef enum MDB_cursor_op {
 	MDB_SET,				/**< Position at specified key */
 	MDB_SET_KEY,			/**< Position at specified key, return key + data */
 	MDB_SET_RANGE,			/**< Position at first key greater than or equal to specified key. */
-	MDB_PREV_MULTIPLE		/**< Position at previous page and return key and up to
+	MDB_PREV_MULTIPLE		/**< Position at previous page and return up to
 								a page of duplicate data items. Only for #MDB_DUPFIXED */
 } MDB_cursor_op;
 

From 4efa54e5f3e5fe08c674b0591910c434d542d0ec Mon Sep 17 00:00:00 2001
From: moneromooo-monero <moneromooo-monero@users.noreply.github.com>
Date: Tue, 15 May 2018 10:53:13 +0100
Subject: [PATCH 367/504] ITS#8857 document mdb_cursor_del does not invalidate
 the cursor

---
 libraries/liblmdb/lmdb.h | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index f427ea58be..57ad1496d2 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -1602,6 +1602,10 @@ int  mdb_cursor_put(MDB_cursor *cursor, MDB_val *key, MDB_val *data,
 	/** @brief Delete current key/data pair
 	 *
 	 * This function deletes the key/data pair to which the cursor refers.
+	 * This does not invalidate the cursor, so operations such as MDB_NEXT
+	 * can still be used on it.
+	 * Both MDB_NEXT and MDB_GET_CURRENT will return the same record after
+	 * this operation.
 	 * @param[in] cursor A cursor handle returned by #mdb_cursor_open()
 	 * @param[in] flags Options for this operation. This parameter
 	 * must be set to 0 or one of the values described here.

From 227840fe251cfbcaff67eebb17a3fa6d635ec144 Mon Sep 17 00:00:00 2001
From: Quanah Gibson-Mount <quanah@openldap.org>
Date: Mon, 14 Jan 2019 18:39:38 +0000
Subject: [PATCH 368/504] Happy New Year!

---
 libraries/liblmdb/midl.c | 2 +-
 libraries/liblmdb/midl.h | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/midl.c b/libraries/liblmdb/midl.c
index 928dcca1cc..fc7da2a732 100644
--- a/libraries/liblmdb/midl.c
+++ b/libraries/liblmdb/midl.c
@@ -3,7 +3,7 @@
 /* $OpenLDAP$ */
 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
  *
- * Copyright 2000-2018 The OpenLDAP Foundation.
+ * Copyright 2000-2019 The OpenLDAP Foundation.
  * Portions Copyright 2001-2018 Howard Chu, Symas Corp.
  * All rights reserved.
  *
diff --git a/libraries/liblmdb/midl.h b/libraries/liblmdb/midl.h
index 474d521aeb..cccb5c45c4 100644
--- a/libraries/liblmdb/midl.h
+++ b/libraries/liblmdb/midl.h
@@ -11,7 +11,7 @@
 /* $OpenLDAP$ */
 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
  *
- * Copyright 2000-2018 The OpenLDAP Foundation.
+ * Copyright 2000-2019 The OpenLDAP Foundation.
  * Portions Copyright 2001-2018 Howard Chu, Symas Corp.
  * All rights reserved.
  *

From 0a202f2ac7c1538b26ef9caf691146872b564364 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 14 Jan 2019 19:06:20 +0000
Subject: [PATCH 369/504] Happy New Year

---
 libraries/liblmdb/COPYRIGHT | 2 +-
 libraries/liblmdb/lmdb.h    | 2 +-
 libraries/liblmdb/mdb.c     | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/COPYRIGHT b/libraries/liblmdb/COPYRIGHT
index 10d7b3204f..f076556eb5 100644
--- a/libraries/liblmdb/COPYRIGHT
+++ b/libraries/liblmdb/COPYRIGHT
@@ -1,4 +1,4 @@
-Copyright 2011-2018 Howard Chu, Symas Corp.
+Copyright 2011-2019 Howard Chu, Symas Corp.
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 57ad1496d2..cf47dbc5ff 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -136,7 +136,7 @@
  *
  *	@author	Howard Chu, Symas Corporation.
  *
- *	@copyright Copyright 2011-2018 Howard Chu, Symas Corp. All rights reserved.
+ *	@copyright Copyright 2011-2019 Howard Chu, Symas Corp. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted only as authorized by the OpenLDAP
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index ad4a3fcb01..e81bd415b4 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5,7 +5,7 @@
  *	BerkeleyDB API, but much simplified.
  */
 /*
- * Copyright 2011-2018 Howard Chu, Symas Corp.
+ * Copyright 2011-2019 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without

From 09fb1de05e956e5e51893dae328e118cdadb80d1 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 17 Jan 2019 18:46:10 +0000
Subject: [PATCH 370/504] IDLs for VL32 must be same size as for 64bit

---
 libraries/liblmdb/midl.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/midl.h b/libraries/liblmdb/midl.h
index cccb5c45c4..202f569bde 100644
--- a/libraries/liblmdb/midl.h
+++ b/libraries/liblmdb/midl.h
@@ -12,7 +12,7 @@
 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
  *
  * Copyright 2000-2019 The OpenLDAP Foundation.
- * Portions Copyright 2001-2018 Howard Chu, Symas Corp.
+ * Portions Copyright 2001-2019 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without

From a51fb486d8cb0c569ad87ce3a506f2df1fc723a1 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 30 Jan 2019 23:43:34 +0000
Subject: [PATCH 371/504] ITS#8969 tweak mdb_page_split

Bump up number of keys for which we use fine-grained splitpoint search
---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index e81bd415b4..13be4b128f 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -10167,7 +10167,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 			 * the split so the new page is emptier than the old page.
 			 * This yields better packing during sequential inserts.
 			 */
-			if (nkeys < 20 || nsize > pmax/16 || newindx >= nkeys) {
+			if (nkeys < 32 || nsize > pmax/16 || newindx >= nkeys) {
 				/* Find split point */
 				psize = 0;
 				if (newindx <= split_indx || newindx >= nkeys) {

From 29f33e7b3a0be77af2cb7cfc5fdd9e0668dc36e9 Mon Sep 17 00:00:00 2001
From: Ka Ho Ng <khng300@gmail.com>
Date: Sun, 17 Feb 2019 23:59:56 +0800
Subject: [PATCH 372/504] ITS#8978 Fix mdb_env_open2() failing when getting
 handle for NTDLL.dll

Always call GetModuleHandleW() with Unicode string, as mdb_fopen() is
calling CreateFileW() already.
---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 13be4b128f..68b754905b 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5053,7 +5053,7 @@ mdb_env_open2(MDB_env *env, int prev)
 		env->me_pidquery = PROCESS_QUERY_INFORMATION;
 	/* Grab functions we need from NTDLL */
 	if (!NtCreateSection) {
-		HMODULE h = GetModuleHandle("NTDLL.DLL");
+		HMODULE h = GetModuleHandleW(L"NTDLL.DLL");
 		if (!h)
 			return MDB_PROBLEM;
 		NtClose = (NtCloseFunc *)GetProcAddress(h, "NtClose");

From 52fbc389259ff9c68150740a00c07c70a32c0551 Mon Sep 17 00:00:00 2001
From: Kris Zyp <kriszyp@gmail.com>
Date: Mon, 15 Apr 2019 20:07:56 -0600
Subject: [PATCH 373/504] ITS#9007 don't free loose writemap pages

Broken in ITS#8756
---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 68b754905b..a621046167 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -3693,9 +3693,9 @@ mdb_freelist_save(MDB_txn *txn)
 			} else {
 				x = mdb_mid2l_search(dl, mp->mp_pgno);
 				mdb_tassert(txn, dl[x].mid == mp->mp_pgno);
+				mdb_dpage_free(env, mp);
 			}
 			dl[x].mptr = NULL;
-			mdb_dpage_free(env, mp);
 		}
 		{
 			/* squash freed slots out of the dirty list */

From cf074e6c1bee7cb0514796c2764ef1d649fb34b2 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 26 Aug 2019 17:51:53 +0100
Subject: [PATCH 374/504] ITS#9068 fix backslash escaping

mdb_load wasn't properly inserting escaped backslashes into the data.
mdb_dump wasn't escaping backslashes when generating printable output.
---
 libraries/liblmdb/mdb_dump.c | 2 ++
 libraries/liblmdb/mdb_load.c | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb_dump.c b/libraries/liblmdb/mdb_dump.c
index 87619bf49a..4942d41abe 100644
--- a/libraries/liblmdb/mdb_dump.c
+++ b/libraries/liblmdb/mdb_dump.c
@@ -64,6 +64,8 @@ static void text(MDB_val *v)
 	end = c + v->mv_size;
 	while (c < end) {
 		if (isprint(*c)) {
+			if (*c == '\\')
+				putchar('\\');
 			putchar(*c);
 		} else {
 			putchar('\\');
diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c
index d313813e31..0a2c91f40a 100644
--- a/libraries/liblmdb/mdb_load.c
+++ b/libraries/liblmdb/mdb_load.c
@@ -236,7 +236,7 @@ badend:
 		while (c2 < end) {
 			if (*c2 == '\\') {
 				if (c2[1] == '\\') {
-					c1++; c2 += 2;
+					*c1++ = *c2;
 				} else {
 					if (c2+3 > end || !isxdigit(c2[1]) || !isxdigit(c2[2])) {
 						Eof = 1;
@@ -244,8 +244,8 @@ badend:
 						return EOF;
 					}
 					*c1++ = unhex(++c2);
-					c2 += 2;
 				}
+				c2 += 2;
 			} else {
 				/* copies are redundant when no escapes were used */
 				*c1++ = *c2++;

From 4bdf8bf5e37b6bd04b6493949facc0686a0cde8c Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sun, 10 Nov 2019 05:00:47 +0000
Subject: [PATCH 375/504] ITS#8704 add missing opt flags in prev commit

---
 libraries/liblmdb/mdb_dump.c | 2 +-
 libraries/liblmdb/mdb_stat.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb_dump.c b/libraries/liblmdb/mdb_dump.c
index 4942d41abe..ef296ce1d1 100644
--- a/libraries/liblmdb/mdb_dump.c
+++ b/libraries/liblmdb/mdb_dump.c
@@ -181,7 +181,7 @@ int main(int argc, char *argv[])
 	 * -V: print version and exit
 	 * (default) dump only the main DB
 	 */
-	while ((i = getopt(argc, argv, "af:lnps:V")) != EOF) {
+	while ((i = getopt(argc, argv, "af:lnps:vV")) != EOF) {
 		switch(i) {
 		case 'V':
 			printf("%s\n", MDB_VERSION_STRING);
diff --git a/libraries/liblmdb/mdb_stat.c b/libraries/liblmdb/mdb_stat.c
index a7f7eff450..54a6257a20 100644
--- a/libraries/liblmdb/mdb_stat.c
+++ b/libraries/liblmdb/mdb_stat.c
@@ -65,7 +65,7 @@ int main(int argc, char *argv[])
 	 * -V: print version and exit
 	 * (default) print stat of only the main DB
 	 */
-	while ((i = getopt(argc, argv, "Vaefnrs:")) != EOF) {
+	while ((i = getopt(argc, argv, "Vaefnrs:v")) != EOF) {
 		switch(i) {
 		case 'V':
 			printf("%s\n", MDB_VERSION_STRING);

From 37e99c2b9d6baf7a3319a7ffa7ab5902add589dc Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Fri, 15 Nov 2019 16:06:34 +0000
Subject: [PATCH 376/504] ITS#9118 add MAP_NOSYNC for FreeBSD

---
 libraries/liblmdb/mdb.c | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index a621046167..13afa0c37a 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -4707,9 +4707,15 @@ mdb_env_map(MDB_env *env, void *addr)
 		return mdb_nt2win32(rc);
 	env->me_map = map;
 #else	/* !_WIN32 */
+	int mmap_flags = MAP_SHARED;
+	int prot = PROT_READ;
+#ifdef MAP_NOSYNC	/* Used on FreeBSD */
+	if (flags & MDB_NOSYNC)
+		mmap_flags |= MAP_NOSYNC;
+#endif
 	if (MDB_REMAPPING(env->me_flags)) {
 		(void) flags;
-		env->me_map = mmap(addr, NUM_METAS * env->me_psize, PROT_READ, MAP_SHARED,
+		env->me_map = mmap(addr, NUM_METAS * env->me_psize, prot, mmap_flags,
 			env->me_fd, 0);
 		if (env->me_map == MAP_FAILED) {
 			env->me_map = NULL;
@@ -4717,13 +4723,12 @@ mdb_env_map(MDB_env *env, void *addr)
 		}
 	} else
 	{
-	int prot = PROT_READ;
 	if (flags & MDB_WRITEMAP) {
 		prot |= PROT_WRITE;
 		if (ftruncate(env->me_fd, env->me_mapsize) < 0)
 			return ErrCode();
 	}
-	env->me_map = mmap(addr, env->me_mapsize, prot, MAP_SHARED,
+	env->me_map = mmap(addr, env->me_mapsize, prot, mmap_flags,
 		env->me_fd, 0);
 	if (env->me_map == MAP_FAILED) {
 		env->me_map = NULL;

From 697d52b6b7532cbef67f93f0b30bcdf97e7cd13a Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Fri, 15 Nov 2019 16:07:51 +0000
Subject: [PATCH 377/504] Silence spurious fallthru warning

---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 13afa0c37a..83b18ecd28 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -8184,7 +8184,7 @@ more:
 						offset *= 4; /* space for 4 more */
 						break;
 					}
-					/* FALLTHRU: Big enough MDB_DUPFIXED sub-page */
+					/* FALLTHRU */ /* Big enough MDB_DUPFIXED sub-page */
 				case MDB_CURRENT:
 					COPY_PGNO(fp->mp_pgno, mp->mp_pgno);
 					mc->mc_xcursor->mx_cursor.mc_pg[0] = fp;

From 1cd0da08a7b695d309e3456fab78a77b5c75bf78 Mon Sep 17 00:00:00 2001
From: Quanah Gibson-Mount <quanah@openldap.org>
Date: Thu, 9 Jan 2020 16:32:20 +0000
Subject: [PATCH 378/504] Happy New Year!

---
 libraries/liblmdb/COPYRIGHT      | 2 +-
 libraries/liblmdb/intro.doc      | 2 +-
 libraries/liblmdb/lmdb.h         | 6 +++---
 libraries/liblmdb/mdb.c          | 2 +-
 libraries/liblmdb/mdb_copy.1     | 2 +-
 libraries/liblmdb/mdb_copy.c     | 2 +-
 libraries/liblmdb/mdb_drop.1     | 2 +-
 libraries/liblmdb/mdb_drop.c     | 2 +-
 libraries/liblmdb/mdb_dump.1     | 2 +-
 libraries/liblmdb/mdb_dump.c     | 2 +-
 libraries/liblmdb/mdb_load.1     | 2 +-
 libraries/liblmdb/mdb_load.c     | 2 +-
 libraries/liblmdb/mdb_stat.1     | 2 +-
 libraries/liblmdb/mdb_stat.c     | 2 +-
 libraries/liblmdb/midl.c         | 4 ++--
 libraries/liblmdb/midl.h         | 4 ++--
 libraries/liblmdb/mtest.c        | 2 +-
 libraries/liblmdb/mtest2.c       | 2 +-
 libraries/liblmdb/mtest3.c       | 2 +-
 libraries/liblmdb/mtest4.c       | 2 +-
 libraries/liblmdb/mtest5.c       | 2 +-
 libraries/liblmdb/mtest6.c       | 2 +-
 libraries/liblmdb/sample-bdb.txt | 2 +-
 libraries/liblmdb/sample-mdb.txt | 2 +-
 24 files changed, 28 insertions(+), 28 deletions(-)

diff --git a/libraries/liblmdb/COPYRIGHT b/libraries/liblmdb/COPYRIGHT
index f076556eb5..d9118b97c9 100644
--- a/libraries/liblmdb/COPYRIGHT
+++ b/libraries/liblmdb/COPYRIGHT
@@ -1,4 +1,4 @@
-Copyright 2011-2019 Howard Chu, Symas Corp.
+Copyright 2011-2020 Howard Chu, Symas Corp.
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/intro.doc b/libraries/liblmdb/intro.doc
index 64dfcaad84..4853af736f 100644
--- a/libraries/liblmdb/intro.doc
+++ b/libraries/liblmdb/intro.doc
@@ -1,5 +1,5 @@
 /*
- * Copyright 2015-2018 Howard Chu, Symas Corp.
+ * Copyright 2015-2020 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index cf47dbc5ff..67b6667127 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -136,7 +136,7 @@
  *
  *	@author	Howard Chu, Symas Corporation.
  *
- *	@copyright Copyright 2011-2019 Howard Chu, Symas Corp. All rights reserved.
+ *	@copyright Copyright 2011-2020 Howard Chu, Symas Corp. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted only as authorized by the OpenLDAP
@@ -234,7 +234,7 @@ typedef int mdb_filehandle_t;
 /** Library minor version */
 #define MDB_VERSION_MINOR	9
 /** Library patch version */
-#define MDB_VERSION_PATCH	70
+#define MDB_VERSION_PATCH	90
 
 /** Combine args a,b,c into a single integer for easy version comparisons */
 #define MDB_VERINT(a,b,c)	(((a) << 24) | ((b) << 16) | (c))
@@ -244,7 +244,7 @@ typedef int mdb_filehandle_t;
 	MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH)
 
 /** The release date of this library version */
-#define MDB_VERSION_DATE	"December 19, 2015"
+#define MDB_VERSION_DATE	"May 1, 2017"
 
 /** A stringifier for the version info */
 #define MDB_VERSTR(a,b,c,d)	"LMDB " #a "." #b "." #c ": (" d ")"
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 83b18ecd28..6323956e51 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5,7 +5,7 @@
  *	BerkeleyDB API, but much simplified.
  */
 /*
- * Copyright 2011-2019 Howard Chu, Symas Corp.
+ * Copyright 2011-2020 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mdb_copy.1 b/libraries/liblmdb/mdb_copy.1
index d5a95014a2..fdf266368d 100644
--- a/libraries/liblmdb/mdb_copy.1
+++ b/libraries/liblmdb/mdb_copy.1
@@ -1,5 +1,5 @@
 .TH MDB_COPY 1 "2017/07/31" "LMDB 0.9.90"
-.\" Copyright 2012-2018 Howard Chu, Symas Corp. All Rights Reserved.
+.\" Copyright 2012-2020 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_copy \- LMDB environment copy tool
diff --git a/libraries/liblmdb/mdb_copy.c b/libraries/liblmdb/mdb_copy.c
index 23be815062..cfcd13ac30 100644
--- a/libraries/liblmdb/mdb_copy.c
+++ b/libraries/liblmdb/mdb_copy.c
@@ -1,6 +1,6 @@
 /* mdb_copy.c - memory-mapped database backup tool */
 /*
- * Copyright 2012-2018 Howard Chu, Symas Corp.
+ * Copyright 2012-2020 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mdb_drop.1 b/libraries/liblmdb/mdb_drop.1
index 370d89c032..ff83fd98b5 100644
--- a/libraries/liblmdb/mdb_drop.1
+++ b/libraries/liblmdb/mdb_drop.1
@@ -1,5 +1,5 @@
 .TH MDB_DROP 1 "2017/11/19" "LMDB 0.9.90"
-.\" Copyright 2014-2018 Howard Chu, Symas Corp. All Rights Reserved.
+.\" Copyright 2014-2020 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_drop \- LMDB database delete tool
diff --git a/libraries/liblmdb/mdb_drop.c b/libraries/liblmdb/mdb_drop.c
index 7d2af4b62e..3d9d779b5b 100644
--- a/libraries/liblmdb/mdb_drop.c
+++ b/libraries/liblmdb/mdb_drop.c
@@ -1,6 +1,6 @@
 /* mdb_drop.c - memory-mapped database delete tool */
 /*
- * Copyright 2016-2018 Howard Chu, Symas Corp.
+ * Copyright 2016-2020 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mdb_dump.1 b/libraries/liblmdb/mdb_dump.1
index 65aafbacdc..bde2773300 100644
--- a/libraries/liblmdb/mdb_dump.1
+++ b/libraries/liblmdb/mdb_dump.1
@@ -1,5 +1,5 @@
 .TH MDB_DUMP 1 "2017/07/31" "LMDB 0.9.90"
-.\" Copyright 2014-2018 Howard Chu, Symas Corp. All Rights Reserved.
+.\" Copyright 2014-2020 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_dump \- LMDB environment export tool
diff --git a/libraries/liblmdb/mdb_dump.c b/libraries/liblmdb/mdb_dump.c
index ef296ce1d1..e0cca9ad25 100644
--- a/libraries/liblmdb/mdb_dump.c
+++ b/libraries/liblmdb/mdb_dump.c
@@ -1,6 +1,6 @@
 /* mdb_dump.c - memory-mapped database dump tool */
 /*
- * Copyright 2011-2018 Howard Chu, Symas Corp.
+ * Copyright 2011-2020 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mdb_load.1 b/libraries/liblmdb/mdb_load.1
index 7bfaa9126c..7dd8aa1f42 100644
--- a/libraries/liblmdb/mdb_load.1
+++ b/libraries/liblmdb/mdb_load.1
@@ -1,5 +1,5 @@
 .TH MDB_LOAD 1 "2015/09/30" "LMDB 0.9.90"
-.\" Copyright 2014-2018 Howard Chu, Symas Corp. All Rights Reserved.
+.\" Copyright 2014-2020 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_load \- LMDB environment import tool
diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c
index 0a2c91f40a..0a99e177f1 100644
--- a/libraries/liblmdb/mdb_load.c
+++ b/libraries/liblmdb/mdb_load.c
@@ -1,6 +1,6 @@
 /* mdb_load.c - memory-mapped database load tool */
 /*
- * Copyright 2011-2018 Howard Chu, Symas Corp.
+ * Copyright 2011-2020 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mdb_stat.1 b/libraries/liblmdb/mdb_stat.1
index f5dd86e228..a618450eff 100644
--- a/libraries/liblmdb/mdb_stat.1
+++ b/libraries/liblmdb/mdb_stat.1
@@ -1,5 +1,5 @@
 .TH MDB_STAT 1 "2017/07/31" "LMDB 0.9.90"
-.\" Copyright 2012-2018 Howard Chu, Symas Corp. All Rights Reserved.
+.\" Copyright 2012-2020 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_stat \- LMDB environment status tool
diff --git a/libraries/liblmdb/mdb_stat.c b/libraries/liblmdb/mdb_stat.c
index 54a6257a20..ad1fef5052 100644
--- a/libraries/liblmdb/mdb_stat.c
+++ b/libraries/liblmdb/mdb_stat.c
@@ -1,6 +1,6 @@
 /* mdb_stat.c - memory-mapped database status tool */
 /*
- * Copyright 2011-2018 Howard Chu, Symas Corp.
+ * Copyright 2011-2020 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/midl.c b/libraries/liblmdb/midl.c
index fc7da2a732..049009e0da 100644
--- a/libraries/liblmdb/midl.c
+++ b/libraries/liblmdb/midl.c
@@ -3,8 +3,8 @@
 /* $OpenLDAP$ */
 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
  *
- * Copyright 2000-2019 The OpenLDAP Foundation.
- * Portions Copyright 2001-2018 Howard Chu, Symas Corp.
+ * Copyright 2000-2020 The OpenLDAP Foundation.
+ * Portions Copyright 2001-2020 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/midl.h b/libraries/liblmdb/midl.h
index 202f569bde..48f4d82d23 100644
--- a/libraries/liblmdb/midl.h
+++ b/libraries/liblmdb/midl.h
@@ -11,8 +11,8 @@
 /* $OpenLDAP$ */
 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
  *
- * Copyright 2000-2019 The OpenLDAP Foundation.
- * Portions Copyright 2001-2019 Howard Chu, Symas Corp.
+ * Copyright 2000-2020 The OpenLDAP Foundation.
+ * Portions Copyright 2001-2020 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest.c b/libraries/liblmdb/mtest.c
index 6fc5840c3f..c1c9abb8f7 100644
--- a/libraries/liblmdb/mtest.c
+++ b/libraries/liblmdb/mtest.c
@@ -1,6 +1,6 @@
 /* mtest.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2018 Howard Chu, Symas Corp.
+ * Copyright 2011-2020 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest2.c b/libraries/liblmdb/mtest2.c
index 64b742aa46..db32525c5b 100644
--- a/libraries/liblmdb/mtest2.c
+++ b/libraries/liblmdb/mtest2.c
@@ -1,6 +1,6 @@
 /* mtest2.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2018 Howard Chu, Symas Corp.
+ * Copyright 2011-2020 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest3.c b/libraries/liblmdb/mtest3.c
index 81e4bbf9b0..bc471eeeaa 100644
--- a/libraries/liblmdb/mtest3.c
+++ b/libraries/liblmdb/mtest3.c
@@ -1,6 +1,6 @@
 /* mtest3.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2018 Howard Chu, Symas Corp.
+ * Copyright 2011-2020 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest4.c b/libraries/liblmdb/mtest4.c
index c355cf105a..b7531755a9 100644
--- a/libraries/liblmdb/mtest4.c
+++ b/libraries/liblmdb/mtest4.c
@@ -1,6 +1,6 @@
 /* mtest4.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2018 Howard Chu, Symas Corp.
+ * Copyright 2011-2020 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest5.c b/libraries/liblmdb/mtest5.c
index 95793ec1fb..d6d1cf9cd7 100644
--- a/libraries/liblmdb/mtest5.c
+++ b/libraries/liblmdb/mtest5.c
@@ -1,6 +1,6 @@
 /* mtest5.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2018 Howard Chu, Symas Corp.
+ * Copyright 2011-2020 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest6.c b/libraries/liblmdb/mtest6.c
index cb0d4d73c9..e4d4e6b27e 100644
--- a/libraries/liblmdb/mtest6.c
+++ b/libraries/liblmdb/mtest6.c
@@ -1,6 +1,6 @@
 /* mtest6.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2018 Howard Chu, Symas Corp.
+ * Copyright 2011-2020 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/sample-bdb.txt b/libraries/liblmdb/sample-bdb.txt
index 97220f0ed2..c72078c722 100644
--- a/libraries/liblmdb/sample-bdb.txt
+++ b/libraries/liblmdb/sample-bdb.txt
@@ -3,7 +3,7 @@
  * Do a line-by-line comparison of this and sample-mdb.txt
  */
 /*
- * Copyright 2012-2018 Howard Chu, Symas Corp.
+ * Copyright 2012-2020 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/sample-mdb.txt b/libraries/liblmdb/sample-mdb.txt
index 1d20ed3d02..e54a847068 100644
--- a/libraries/liblmdb/sample-mdb.txt
+++ b/libraries/liblmdb/sample-mdb.txt
@@ -3,7 +3,7 @@
  * Do a line-by-line comparison of this and sample-bdb.txt
  */
 /*
- * Copyright 2012-2018 Howard Chu, Symas Corp.
+ * Copyright 2012-2020 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without

From b485f2869c39a5e516e1940b90d44e4192683b26 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 23 Jan 2020 14:50:00 +0000
Subject: [PATCH 379/504] ITS#9155 lmdb: free mt_spill_pgs in non-nested txn on
 end

---
 libraries/liblmdb/mdb.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 6323956e51..603267b9b1 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -3570,12 +3570,12 @@ mdb_txn_end(MDB_txn *txn, unsigned mode)
 			txn->mt_parent->mt_flags &= ~MDB_TXN_HAS_CHILD;
 			env->me_pgstate = ((MDB_ntxn *)txn)->mnt_pgstate;
 			mdb_midl_free(txn->mt_free_pgs);
-			mdb_midl_free(txn->mt_spill_pgs);
-#if OVERFLOW_NOTYET
-			mdb_mid2l_free(txn->mt_dirty_ovs);
-#endif
 			free(txn->mt_u.dirty_list);
 		}
+		mdb_midl_free(txn->mt_spill_pgs);
+#if OVERFLOW_NOTYET
+		mdb_mid2l_free(txn->mt_dirty_ovs);
+#endif
 
 		mdb_midl_free(pghead);
 	}

From dfb3bbed656132456001c5aaca246fd4430e5ef5 Mon Sep 17 00:00:00 2001
From: Kris Zyp <kriszyp@gmail.com>
Date: Tue, 18 Feb 2020 22:49:03 -0700
Subject: [PATCH 380/504] ITS#9017 LMDB: improve Windows sync commit perf

---
 libraries/liblmdb/mdb.c | 235 +++++++++++++++++++++++++++-------------
 1 file changed, 162 insertions(+), 73 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 603267b9b1..e0b61e1c4f 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1565,9 +1565,12 @@ struct MDB_env {
 	HANDLE		me_fd;		/**< The main data file */
 	HANDLE		me_lfd;		/**< The lock file */
 	HANDLE		me_mfd;		/**< For writing and syncing the meta pages */
-#if MDB_RPAGE_CACHE && defined(_WIN32)
+#ifdef _WIN32
+#ifdef MDB_RPAGE_CACHE
 	HANDLE		me_fmh;		/**< File Mapping handle */
 #endif
+	HANDLE		me_ovfd;	/**< Overlapped/async with write-through file handle */
+#endif /* _WIN32 */
 	/** Failed to update the meta page. Probably an I/O error. */
 #define	MDB_FATAL_ERROR	0x80000000U
 	/** Some fields are initialized. */
@@ -1620,6 +1623,8 @@ struct MDB_env {
 	int		me_live_reader;		/**< have liveness lock in reader table */
 #ifdef _WIN32
 	int		me_pidquery;		/**< Used in OpenProcess */
+	OVERLAPPED	*ov;			/**< Used for for overlapping I/O requests */
+	int		ovs;				/**< Count of OVERLAPPEDs */
 #endif
 #ifdef MDB_USE_POSIX_MUTEX	/* Posix mutexes reside in shared mem */
 #	define		me_rmutex	me_txns->mti_rmutex /**< Shared reader lock */
@@ -2977,7 +2982,11 @@ mdb_env_sync0(MDB_env *env, int force, pgno_t numpgs)
 	int rc = 0;
 	if (env->me_flags & MDB_RDONLY)
 		return EACCES;
-	if (force || !F_ISSET(env->me_flags, MDB_NOSYNC)) {
+	if (force
+#ifndef _WIN32	/* Sync is normally achieved in Windows by doing WRITE_THROUGH writes */
+		|| !(env->me_flags & MDB_NOSYNC)
+#endif
+		) {
 		if (env->me_flags & MDB_WRITEMAP) {
 			int flags = ((env->me_flags & MDB_MAPASYNC) && !force)
 				? MS_ASYNC : MS_SYNC;
@@ -3907,19 +3916,50 @@ mdb_page_flush(MDB_txn *txn, int keep)
 #endif
 #ifdef _WIN32
 	OVERLAPPED	ov;
+	MDB_page    *wdp;
+	int async_i = 0;
+	HANDLE fd = (env->me_flags & MDB_NOSYNC) ? env->me_fd : env->me_ovfd;
 #else
 	struct iovec iov[MDB_COMMIT_PAGES];
+	HANDLE fd = env->me_fd;
+#endif
 	ssize_t		wsize = 0, wres;
 	off_t		wpos = 0, next_pos = 1; /* impossible pos, so pos != next_pos */
 	int			n = 0;
-#endif
 
 	j = i = keep;
 
-	if (env->me_flags & MDB_WRITEMAP) {
+	if (env->me_flags & MDB_WRITEMAP
+#ifdef _WIN32
+		/* In windows, we still do writes to the file (with write-through enabled in sync mode),
+		 * as this is faster than FlushViewOfFile/FlushFileBuffers */
+		&& (env->me_flags & MDB_NOSYNC)
+#endif
+	) {
 		goto done;
 	}
 
+#ifdef _WIN32
+	if (pagecount - keep >= env->ovs) {
+		/* ran out of room in ov array, and re-malloc, copy handles and free previous */
+		int ovs = (pagecount - keep) * 1.5; /* provide extra padding to reduce number of re-allocations */
+		int new_size = ovs * sizeof(OVERLAPPED);
+		ov = malloc(new_size);
+		if (ov == NULL)
+			return ENOMEM;
+		int previous_size = env->ovs * sizeof(OVERLAPPED);
+		memcpy(ov, env->ov, previous_size); /* Copy previous OVERLAPPED data to retain event handles */
+		/* And clear rest of memory */
+		memset(&ov[env->ovs], 0, new_size - previous_size);
+		if (env->ovs > 0) {
+			free(env->ov); /* release previous allocation */
+		}
+
+		env->ov = ov;
+		env->ovs = ovs;
+	}
+#endif
+
 	/* Write the pages */
 	for (;;) {
 		if (++i <= pagecount) {
@@ -3945,60 +3985,58 @@ mdb_page_flush(MDB_txn *txn, int keep)
 			if (IS_OVERFLOW(dp)) size *= dp->mp_pages;
 #endif
 		}
-#ifdef _WIN32
-		else break;
-
-#if MDB_RPAGE_CACHE
-		if (env->me_encfunc) {
-			MDB_val in, out;
-			encp = mdb_page_malloc(txn, nump. 0);
-			if (!encp)
-				return ENOMEM;
-			in.mv_size = size;
-			in.mv_data = dp;
-			out.mv_size = size;
-			out.mv_data = encp;
-			env->me_encfunc(&in, &out, env->me_enckey, 1);
-			dp = encp;
-		}
-#endif
-		/* Windows actually supports scatter/gather I/O, but only on
-		 * unbuffered file handles. Since we're relying on the OS page
-		 * cache for all our data, that's self-defeating. So we just
-		 * write pages one at a time. We use the ov structure to set
-		 * the write offset, to at least save the overhead of a Seek
-		 * system call.
-		 */
-		DPRINTF(("committing page %"Yu, pgno));
-		memset(&ov, 0, sizeof(ov));
-		ov.Offset = pos & 0xffffffff;
-		ov.OffsetHigh = pos >> 16 >> 16;
-		rc = 0;
-		if (!WriteFile(env->me_fd, dp, size, NULL, &ov))
-			rc = ErrCode();
-#if MDB_RPAGE_CACHE
-		if (env->me_encfunc)
-			mdb_dpage_free_n(env, dp, nump);
-#endif
-		if (rc) {
-			DPRINTF(("WriteFile: %d", rc));
-			return rc;
-		}
-#else
 		/* Write up to MDB_COMMIT_PAGES dirty pages at a time. */
-		if (pos!=next_pos || n==MDB_COMMIT_PAGES || wsize+size>MAX_WRITE) {
+		if (pos!=next_pos || n==MDB_COMMIT_PAGES || wsize+size>MAX_WRITE
+#ifdef _WIN32
+			/* Windows actually supports scatter/gather I/O, but only on
+			 * unbuffered file handles. Since we're relying on the OS page
+			 * cache for all our data, that's self-defeating. So we just
+			 * write pages one at a time. We use the ov structure to set
+			 * the write offset, to at least save the overhead of a Seek
+			 * system call.
+			 * If writemap is enabled, consecutive page positions infer
+			 * contiguous (mapped) memory.
+			 * Otherwise force write pages one at a time.
+			 */
+			|| !(env->me_flags & MDB_WRITEMAP)
+#endif
+			) {
 			if (n) {
 retry_write:
 				rc = 0;
 				/* Write previous page(s) */
+#ifdef _WIN32
+				OVERLAPPED *this_ov = &ov[async_i];
+				/* Clear status, and keep hEvent, we reuse that */
+				this_ov->Internal = 0;
+				this_ov->Offset = wpos & 0xffffffff;
+				this_ov->OffsetHigh = wpos >> 16 >> 16;
+				if (!F_ISSET(env->me_flags, MDB_NOSYNC) && !this_ov->hEvent) {
+					HANDLE event = CreateEvent(NULL, FALSE, FALSE, NULL);
+					if (!event) {
+						rc = ErrCode();
+						DPRINTF(("CreateEvent: %s", strerror(rc)));
+						return rc;
+					}
+					this_ov->hEvent = event;
+				}
+				if (!WriteFile(fd, wdp, wsize, NULL, this_ov)) {
+					rc = ErrCode();
+					if (rc != ERROR_IO_PENDING) {
+						DPRINTF(("WriteFile: %d", rc));
+						return rc;
+					}
+				}
+				async_i++;
+#else /* _WIN32 */
 #ifdef MDB_USE_PWRITEV
-				wres = pwritev(env->me_fd, iov, n, wpos);
+				wres = pwritev(fd, iov, n, wpos);
 #else
 				if (n == 1) {
-					wres = pwrite(env->me_fd, iov[0].iov_base, wsize, wpos);
+					wres = pwrite(fd, iov[0].iov_base, wsize, wpos);
 				} else {
 retry_seek:
-					if (lseek(env->me_fd, wpos, SEEK_SET) == -1) {
+					if (lseek(fd, wpos, SEEK_SET) == -1) {
 						rc = ErrCode();
 						if (rc == EINTR)
 							goto retry_seek;
@@ -4006,7 +4044,7 @@ retry_seek:
 						wres = wsize;
 					} else {
 						rc = 0;
-						wres = writev(env->me_fd, iov, n);
+						wres = writev(fd, iov, n);
 					}
 				}
 #endif
@@ -4021,6 +4059,7 @@ retry_seek:
 						DPUTS("short write, filesystem full?");
 					}
 				}
+#endif /* _WIN32 */
 #if MDB_RPAGE_CACHE
 				if (env->me_encfunc) {
 					int j, num1;
@@ -4053,13 +4092,16 @@ retry_seek:
 			dp = encp;
 		}
 #endif
-		DPRINTF(("committing page %"Yu, pgno));
-		next_pos = pos + size;
+#ifdef _WIN32
+		wdp = dp;
+#else
 		iov[n].iov_len = size;
 		iov[n].iov_base = (char *)dp;
+#endif
+		DPRINTF(("committing page %"Yu, pgno));
+		next_pos = pos + size;
 		wsize += size;
 		n++;
-#endif	/* _WIN32 */
 	}
 #if MDB_RPAGE_CACHE
 	if (MDB_REMAPPING(env->me_flags) && pgno > txn->mt_last_pgno)
@@ -4072,21 +4114,49 @@ retry_seek:
 	 */
 	CACHEFLUSH(env->me_map, txn->mt_next_pgno * env->me_psize, DCACHE);
 
-	for (i = keep; ++i <= pagecount; ) {
-		dp = dl[i].mptr;
-		/* This is a page we skipped above */
-		if (!dl[i].mid) {
-			dl[++j] = dl[i];
-			dl[j].mid = dp->mp_pgno;
-			continue;
+#ifdef _WIN32
+	if (!F_ISSET(env->me_flags, MDB_NOSYNC)) {
+		/* Now wait for all the asynchronous/overlapped sync/write-through writes to complete.
+		* We start with the last one so that all the others should already be complete and
+		* we reduce thread suspend/resuming (in practice, typically about 99.5% of writes are
+		* done after the last write is done) */
+		rc = 0;
+		while (--async_i >= 0) {
+			if (ov[async_i].hEvent) {
+				if (!GetOverlappedResult(fd, &ov[async_i], &wres, TRUE)) {
+					rc = ErrCode(); /* Continue on so that all the event signals are reset */
+				}
+			}
+		}
+		if (rc) { /* any error on GetOverlappedResult, exit now */
+			return rc;
 		}
-		mdb_dpage_free(env, dp);
 	}
+#endif	/* _WIN32 */
+
+    if (!(env->me_flags & MDB_WRITEMAP)) {
+        /* Don't free pages when using writemap (can only get here in NOSYNC mode in Windows)
+         * MIPS has cache coherency issues, this is a no-op everywhere else
+         * Note: for any size >= on-chip cache size, entire on-chip cache is
+         * flushed.
+         */
+		for (i = keep; ++i <= pagecount; ) {
+			dp = dl[i].mptr;
+			/* This is a page we skipped above */
+			if (!dl[i].mid) {
+				dl[++j] = dl[i];
+				dl[j].mid = dp->mp_pgno;
+				continue;
+			}
+			mdb_dpage_free(env, dp);
+		}
+	}
+
+done:
 	i--;
 	txn->mt_dirty_room += i - j;
 	dl[0].mid = j;
 
-done:
 	return MDB_SUCCESS;
 }
 
@@ -4443,7 +4513,6 @@ mdb_env_init_meta(MDB_env *env, MDB_meta *meta)
 	if (len == -1 && ErrCode() == EINTR) continue; \
 	rc = (len >= 0); break; } while(1)
 #endif
-
 	DPUTS("writing new meta page");
 
 	psize = env->me_psize;
@@ -4513,6 +4582,7 @@ mdb_env_write_meta(MDB_txn *txn)
 	if (mapsize < env->me_mapsize)
 		mapsize = env->me_mapsize;
 
+#ifndef _WIN32 /* We don't want to ever use MSYNC/FlushViewOfFile in Windows */
 	if (flags & MDB_WRITEMAP) {
 		mp->mm_mapsize = mapsize;
 		mp->mm_dbs[FREE_DBI] = txn->mt_dbs[FREE_DBI];
@@ -4528,11 +4598,10 @@ mdb_env_write_meta(MDB_txn *txn)
 			unsigned meta_size = env->me_psize;
 			rc = (env->me_flags & MDB_MAPASYNC) ? MS_ASYNC : MS_SYNC;
 			ptr = (char *)mp - PAGEHDRSZ;
-#ifndef _WIN32	/* POSIX msync() requires ptr = start of OS page */
+			/* POSIX msync() requires ptr = start of OS page */
 			r2 = (ptr - env->me_map) & (env->me_os_psize - 1);
 			ptr -= r2;
 			meta_size += r2;
-#endif
 			if (MDB_MSYNC(ptr, meta_size, rc)) {
 				rc = ErrCode();
 				goto fail;
@@ -4540,6 +4609,7 @@ mdb_env_write_meta(MDB_txn *txn)
 		}
 		goto done;
 	}
+#endif
 	metab.mm_txnid = mp->mm_txnid;
 	metab.mm_last_pg = mp->mm_last_pg;
 
@@ -4921,7 +4991,7 @@ mdb_fname_init(const char *path, unsigned envflags, MDB_name *fname)
 /** File type, access mode etc. for #mdb_fopen() */
 enum mdb_fopen_type {
 #ifdef _WIN32
-	MDB_O_RDONLY, MDB_O_RDWR, MDB_O_META, MDB_O_COPY, MDB_O_LOCKS
+	MDB_O_RDONLY, MDB_O_RDWR, MDB_O_OVERLAPPED, MDB_O_META, MDB_O_COPY, MDB_O_LOCKS
 #else
 	/* A comment in mdb_fopen() explains some O_* flag choices. */
 	MDB_O_RDONLY= O_RDONLY,                            /**< for RDONLY me_fd */
@@ -4982,6 +5052,11 @@ mdb_fopen(const MDB_env *env, MDB_name *fname,
 	disp = OPEN_ALWAYS;
 	attrs = FILE_ATTRIBUTE_NORMAL;
 	switch (which) {
+	case MDB_O_OVERLAPPED: 	/* for unbuffered asynchronous writes (write-through mode)*/
+		acc = GENERIC_WRITE;
+		disp = OPEN_EXISTING;
+		attrs = FILE_FLAG_OVERLAPPED|FILE_FLAG_WRITE_THROUGH;
+		break;
 	case MDB_O_RDONLY:			/* read-only datafile */
 		acc = GENERIC_READ;
 		disp = OPEN_EXISTING;
@@ -5071,6 +5146,7 @@ mdb_env_open2(MDB_env *env, int prev)
 		if (!NtCreateSection)
 			return MDB_PROBLEM;
 	}
+	env->ovs = 0;
 #endif /* _WIN32 */
 
 #ifdef BROKEN_FDATASYNC
@@ -5909,6 +5985,11 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 		mode, &env->me_fd);
 	if (rc)
 		goto leave;
+#ifdef _WIN32
+	rc = mdb_fopen(env, &fname, MDB_O_OVERLAPPED, mode, &env->me_ovfd);
+	if (rc)
+		goto leave;
+#endif
 
 	if ((flags & (MDB_RDONLY|MDB_NOLOCK)) == MDB_RDONLY) {
 		rc = mdb_env_setup_locks(env, &fname, mode, &excl);
@@ -5917,14 +5998,12 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 	}
 
 	if ((rc = mdb_env_open2(env, flags & MDB_PREVSNAPSHOT)) == MDB_SUCCESS) {
-		if (!(flags & (MDB_RDONLY|MDB_WRITEMAP))) {
-			/* Synchronous fd for meta writes. Needed even with
-			 * MDB_NOSYNC/MDB_NOMETASYNC, in case these get reset.
-			 */
-			rc = mdb_fopen(env, &fname, MDB_O_META, mode, &env->me_mfd);
-			if (rc)
-				goto leave;
-		}
+		/* Synchronous fd for meta writes. Needed even with
+		 * MDB_NOSYNC/MDB_NOMETASYNC, in case these get reset.
+		 */
+		rc = mdb_fopen(env, &fname, MDB_O_META, mode, &env->me_mfd);
+		if (rc)
+			goto leave;
 		DPRINTF(("opened dbenv %p", (void *) env));
 		if (excl > 0 && !(flags & MDB_PREVSNAPSHOT)) {
 			rc = mdb_env_share_locks(env, &excl);
@@ -6030,6 +6109,16 @@ mdb_env_close_active(MDB_env *env, int excl)
 	}
 	if (env->me_mfd != INVALID_HANDLE_VALUE)
 		(void) close(env->me_mfd);
+#ifdef _WIN32
+	if (env->ovs > 0) {
+		for (i = 0; i < env->ovs; i++) {
+			CloseHandle(env->ov[i].hEvent);
+		}
+		free(env->ov);
+	}
+	if (env->me_ovfd != INVALID_HANDLE_VALUE)
+		(void) close(env->me_ovfd);
+#endif
 	if (env->me_fd != INVALID_HANDLE_VALUE)
 		(void) close(env->me_fd);
 	if (env->me_txns) {

From 485465b6a700bb6a7084c875af6eaded815fb645 Mon Sep 17 00:00:00 2001
From: Kris Zyp <kriszyp@gmail.com>
Date: Tue, 18 Feb 2020 22:50:41 -0700
Subject: [PATCH 381/504] ITS#9017 LMDB: allow using fixed file size on Windows

---
 libraries/liblmdb/mdb.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index e0b61e1c4f..994d843248 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -4760,7 +4760,14 @@ mdb_env_map(MDB_env *env, void *addr)
 		alloctype = MEM_RESERVE;
 	}
 
+#ifdef MDB_FIXEDSIZE
+	LARGE_INTEGER fsize;
+	fsize.LowPart = msize & 0xffffffff;
+	fsize.HighPart = msize >> 16 >> 16;
+	rc = NtCreateSection(&mh, access, NULL, &fsize, secprot, SEC_RESERVE, env->me_fd);
+#else
 	rc = NtCreateSection(&mh, access, NULL, NULL, secprot, SEC_RESERVE, env->me_fd);
+#endif
 	if (rc)
 		return mdb_nt2win32(rc);
 	map = addr;

From 7deed692021cdcf9e06048ab01106a1222441e36 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Fri, 24 Apr 2020 15:03:33 +0100
Subject: [PATCH 382/504] ITS#9017 LMDB: fix off_t usage on Windows

---
 libraries/liblmdb/mdb.c | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 994d843248..af12b51771 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -96,6 +96,7 @@ static NtCloseFunc *NtClose;
 #  define SSIZE_MAX	INT_MAX
 # endif
 #endif
+#define MDB_OFF_T	LARGE_INTEGER
 #else
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -108,6 +109,7 @@ static NtCloseFunc *NtClose;
 #include <sys/file.h>
 #endif
 #include <fcntl.h>
+#define MDB_OFF_T	off_t
 #endif
 
 #if defined(__mips) && defined(__linux)
@@ -1596,7 +1598,7 @@ struct MDB_env {
 	MDB_txn		*me_txn;		/**< current write transaction */
 	MDB_txn		*me_txn0;		/**< prealloc'd write transaction */
 	mdb_size_t	me_mapsize;		/**< size of the data memory map */
-	off_t		me_size;		/**< current file size */
+	MDB_OFF_T	me_size;		/**< current file size */
 	pgno_t		me_maxpg;		/**< me_mapsize / me_psize */
 	MDB_dbx		*me_dbxs;		/**< array of static DB info */
 	uint16_t	*me_dbflags;	/**< array of flags from MDB_db.md_flags */
@@ -3907,7 +3909,7 @@ mdb_page_flush(MDB_txn *txn, int keep)
 	unsigned	psize = env->me_psize, j;
 	int			i, pagecount = dl[0].mid, rc;
 	size_t		size = 0;
-	off_t		pos = 0;
+	MDB_OFF_T	pos = 0;
 	pgno_t		pgno = 0;
 	MDB_page	*dp = NULL;
 #if MDB_RPAGE_CACHE
@@ -3924,7 +3926,7 @@ mdb_page_flush(MDB_txn *txn, int keep)
 	HANDLE fd = env->me_fd;
 #endif
 	ssize_t		wsize = 0, wres;
-	off_t		wpos = 0, next_pos = 1; /* impossible pos, so pos != next_pos */
+	MDB_OFF_T	wpos = 0, next_pos = 1; /* impossible pos, so pos != next_pos */
 	int			n = 0;
 
 	j = i = keep;
@@ -4560,7 +4562,7 @@ mdb_env_write_meta(MDB_txn *txn)
 	MDB_meta	meta, metab, *mp;
 	unsigned flags;
 	mdb_size_t mapsize;
-	off_t off;
+	MDB_OFF_T off;
 	int rc, len, toggle;
 	char *ptr;
 	HANDLE mfd;
@@ -5586,7 +5588,7 @@ mdb_env_setup_locks(MDB_env *env, MDB_name *fname, int mode, int *excl)
 	union semun semu;
 #endif
 	int rc;
-	off_t size, rsize;
+	MDB_OFF_T size, rsize;
 
 	rc = mdb_fopen(env, fname, MDB_O_LOCKS, mode, &env->me_lfd);
 	if (rc) {
@@ -6535,7 +6537,7 @@ mdb_rpage_get(MDB_txn *txn, pgno_t pg0, int numpgs, MDB_page **ret)
 	pgno_t pgno;
 	int rc, retries = 1;
 #ifdef _WIN32
-	LARGE_INTEGER off;
+	MDB_OFF_T off;
 	SIZE_T len;
 #define SET_OFF(off,val)	off.QuadPart = val
 #define MAP(rc,env,addr,len,off)	\
@@ -6544,7 +6546,7 @@ mdb_rpage_get(MDB_txn *txn, pgno_t pg0, int numpgs, MDB_page **ret)
 		len, &off, &len, ViewUnmap, (env->me_flags & MDB_RDONLY) ? 0 : MEM_RESERVE, PAGE_READONLY); \
 	if (rc) rc = mdb_nt2win32(rc)
 #else
-	off_t off;
+	MDB_OFF_T off;
 	size_t len;
 #define SET_OFF(off,val)	off = val
 #define MAP(rc,env,addr,len,off)	\

From 9c01725fae58687e5d012c7c5edbeca05950997a Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Fri, 24 Apr 2020 15:07:33 +0100
Subject: [PATCH 383/504] ITS#9017 doxygen comment for MDB_FIXEDSIZE

---
 libraries/liblmdb/mdb.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index af12b51771..cf84c98e38 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -4762,6 +4762,11 @@ mdb_env_map(MDB_env *env, void *addr)
 		alloctype = MEM_RESERVE;
 	}
 
+	/** Some users are afraid of seeing their disk space getting used
+	 * all at once, so the default is now to do incremental file growth.
+	 * But that has a large performance impact, so give the option of
+	 * allocating the file up front.
+	 */
 #ifdef MDB_FIXEDSIZE
 	LARGE_INTEGER fsize;
 	fsize.LowPart = msize & 0xffffffff;

From 034a7e98a6e7c93c1d04b0b3290fd49d2b861bfc Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sat, 25 Apr 2020 00:46:58 +0100
Subject: [PATCH 384/504] ITS#9017 cleanup Windows off_t

---
 libraries/liblmdb/mdb.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index cf84c98e38..8437459ed3 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -96,7 +96,7 @@ static NtCloseFunc *NtClose;
 #  define SSIZE_MAX	INT_MAX
 # endif
 #endif
-#define MDB_OFF_T	LARGE_INTEGER
+#define MDB_OFF_T	int64_t
 #else
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -6542,7 +6542,7 @@ mdb_rpage_get(MDB_txn *txn, pgno_t pg0, int numpgs, MDB_page **ret)
 	pgno_t pgno;
 	int rc, retries = 1;
 #ifdef _WIN32
-	MDB_OFF_T off;
+	LARGE_INTEGER off;
 	SIZE_T len;
 #define SET_OFF(off,val)	off.QuadPart = val
 #define MAP(rc,env,addr,len,off)	\
@@ -6551,7 +6551,7 @@ mdb_rpage_get(MDB_txn *txn, pgno_t pg0, int numpgs, MDB_page **ret)
 		len, &off, &len, ViewUnmap, (env->me_flags & MDB_RDONLY) ? 0 : MEM_RESERVE, PAGE_READONLY); \
 	if (rc) rc = mdb_nt2win32(rc)
 #else
-	MDB_OFF_T off;
+	off_t off;
 	size_t len;
 #define SET_OFF(off,val)	off = val
 #define MAP(rc,env,addr,len,off)	\

From 8bcc050feee22e45a6dab7b8438d48da267cf7e5 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 16 Jun 2020 19:49:14 +0100
Subject: [PATCH 385/504] ITS#9278 fix robust mutex cleanup for FreeBSD

FreeBSD 11 supports robust process-shared POSIX mutexes,
but requires them to be explicitly destroyed before munmap
---
 libraries/liblmdb/mdb.c | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 8437459ed3..7a15943ac5 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -161,7 +161,10 @@ typedef SSIZE_T	ssize_t;
 #include <resolv.h>	/* defines BYTE_ORDER on HPUX and Solaris */
 #endif
 
-#if defined(__APPLE__) || defined (BSD) || defined(__FreeBSD_kernel__)
+#if defined(__FreeBSD__) && defined(__FreeBSD_version) && __FreeBSD_version >= 1100110
+# define MDB_USE_POSIX_MUTEX	1
+# define MDB_USE_ROBUST	1
+#elif defined(__APPLE__) || defined (BSD) || defined(__FreeBSD_kernel__)
 # if !(defined(MDB_USE_POSIX_MUTEX) || defined(MDB_USE_POSIX_SEM))
 # define MDB_USE_SYSV_SEM	1
 # endif
@@ -1719,7 +1722,7 @@ static int	mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata,
 static int  mdb_env_read_header(MDB_env *env, int prev, MDB_meta *meta);
 static MDB_meta *mdb_env_pick_meta(const MDB_env *env);
 static int  mdb_env_write_meta(MDB_txn *txn);
-#ifdef MDB_USE_POSIX_MUTEX /* Drop unused excl arg */
+#if defined(MDB_USE_POSIX_MUTEX) && !defined(MDB_ROBUST_SUPPORTED) /* Drop unused excl arg */
 # define mdb_env_close_active(env, excl) mdb_env_close1(env)
 #endif
 static void mdb_env_close_active(MDB_env *env, int excl);
@@ -6180,6 +6183,17 @@ mdb_env_close_active(MDB_env *env, int excl)
 			if (excl > 0)
 				semctl(env->me_rmutex->semid, 0, IPC_RMID);
 		}
+#elif defined(MDB_ROBUST_SUPPORTED)
+		/* If we have the filelock:  If we are the
+		 * only remaining user, clean up robust
+		 * mutexes.
+		 */
+		if (excl == 0)
+			mdb_env_excl_lock(env, &excl);
+		if (excl > 0) {
+			pthread_mutex_destroy(env->me_txns->mti_rmutex);
+			pthread_mutex_destroy(env->me_txns->mti_wmutex);
+		}
 #endif
 		munmap((void *)env->me_txns, (env->me_maxreaders-1)*sizeof(MDB_reader)+sizeof(MDB_txninfo));
 	}

From 5c0dda76c96a3badc9c61fe840c7098451c7ffa2 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sun, 21 Feb 2016 18:29:40 +0000
Subject: [PATCH 386/504] Preliminary raw partition support

Autodetects that a block device is being used.
---
 libraries/liblmdb/mdb.c | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 7a15943ac5..df79f39852 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1584,6 +1584,8 @@ struct MDB_env {
 #define	MDB_ENV_TXKEY	0x10000000U
 	/** fdatasync is unreliable */
 #define	MDB_FSYNCONLY	0x08000000U
+	/** using a raw block device */
+#define	MDB_RAWPART		0x04000000U
 	uint32_t 	me_flags;		/**< @ref mdb_env */
 	unsigned int	me_psize;	/**< DB page size, inited from me_os_psize */
 	unsigned int	me_os_psize;	/**< OS page size, from #GET_PAGESIZE */
@@ -4456,6 +4458,8 @@ mdb_env_read_header(MDB_env *env, int prev, MDB_meta *meta)
 		p = (MDB_page *)&pbuf;
 
 		if (!F_ISSET(p->mp_flags, P_META)) {
+			if (env->me_flags & MDB_RAWPART)
+				return ENOENT;
 			DPRINTF(("page %"Yu" not a meta page", p->mp_pgno));
 			return MDB_INVALID;
 		}
@@ -4812,7 +4816,7 @@ mdb_env_map(MDB_env *env, void *addr)
 	{
 	if (flags & MDB_WRITEMAP) {
 		prot |= PROT_WRITE;
-		if (ftruncate(env->me_fd, env->me_mapsize) < 0)
+		if (!(flags & MDB_RAWPART) && ftruncate(env->me_fd, env->me_mapsize) < 0)
 			return ErrCode();
 	}
 	env->me_map = mmap(addr, env->me_mapsize, prot, mmap_flags,
@@ -5945,6 +5949,17 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 		goto leave;
 #endif
 	}
+#endif
+#ifndef _WIN32
+	{
+		struct stat st;
+		rc = stat(path, &st);
+		if (rc)
+			return ErrCode();
+		flags &= ~MDB_RAWPART;
+		if (S_ISBLK(st.st_mode))
+			flags |= MDB_RAWPART | MDB_NOSUBDIR;
+	}
 #endif
 	flags |= MDB_ENV_ACTIVE; /* tell mdb_env_close_active() to clean up */
 

From a7df9e63a5097b7915a3b4a2788c1166e3c1b470 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 11 Jun 2018 10:12:03 +0100
Subject: [PATCH 387/504] More RAWPART support

Use mmap to read and initialize the meta pages, raw device
may not support read/write syscalls.
---
 libraries/liblmdb/mdb.c | 40 ++++++++++++++++++++++++++++++++++++++--
 1 file changed, 38 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index df79f39852..5b22841242 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -4415,6 +4415,8 @@ fail:
 	return rc;
 }
 
+static int ESECT mdb_env_map(MDB_env *env, void *addr);
+
 /** Read the environment parameters of a DB environment before
  * mapping it into memory.
  * @param[in] env the environment handle
@@ -4431,6 +4433,27 @@ mdb_env_read_header(MDB_env *env, int prev, MDB_meta *meta)
 	int			i, rc, off;
 	enum { Size = sizeof(pbuf) };
 
+	if (env->me_flags & MDB_RAWPART) {
+#define VM_ALIGN	0x200000
+		env->me_mapsize += VM_ALIGN-1;
+		env->me_mapsize &= ~(VM_ALIGN-1);
+		env->me_psize = env->me_os_psize;
+		rc = mdb_env_map(env, NULL);
+		p = (MDB_page *)env->me_map;
+		for (i=0; i<NUM_METAS; i++) {
+			if (!F_ISSET(p->mp_flags, P_META))
+				return ENOENT;
+			if (env->me_metas[i]->mm_magic != MDB_MAGIC)
+				return MDB_INVALID;
+			if (env->me_metas[i]->mm_version != MDB_DATA_VERSION)
+				return MDB_VERSION_MISMATCH;
+			if (i == 0 || env->me_metas[i]->mm_txnid > meta->mm_txnid)
+				*meta = *env->me_metas[i];
+			p = (MDB_page *)((char *)p + env->me_psize);
+		}
+		return 0;
+	}
+
 	/* We don't know the page size yet, so use a minimum value.
 	 * Read both meta pages so we can use the latest one.
 	 */
@@ -4526,6 +4549,18 @@ mdb_env_init_meta(MDB_env *env, MDB_meta *meta)
 
 	psize = env->me_psize;
 
+	if ((env->me_flags & (MDB_RAWPART|MDB_WRITEMAP)) == (MDB_RAWPART|MDB_WRITEMAP)) {
+		p = (MDB_page *)env->me_map;
+		p->mp_pgno = 0;
+		p->mp_flags = P_META;
+		*(MDB_meta *)METADATA(p) = *meta;
+		q = (MDB_page *)((char *)p + psize);
+		q->mp_pgno = 1;
+		q->mp_flags = P_META;
+		*(MDB_meta *)METADATA(q) = *meta;
+		return 0;
+	}
+
 	p = calloc(NUM_METAS, psize);
 	if (!p)
 		return ENOMEM;
@@ -4800,6 +4835,8 @@ mdb_env_map(MDB_env *env, void *addr)
 #else	/* !_WIN32 */
 	int mmap_flags = MAP_SHARED;
 	int prot = PROT_READ;
+	if (flags & MDB_WRITEMAP)
+		prot |= PROT_WRITE;
 #ifdef MAP_NOSYNC	/* Used on FreeBSD */
 	if (flags & MDB_NOSYNC)
 		mmap_flags |= MAP_NOSYNC;
@@ -4815,7 +4852,6 @@ mdb_env_map(MDB_env *env, void *addr)
 	} else
 	{
 	if (flags & MDB_WRITEMAP) {
-		prot |= PROT_WRITE;
 		if (!(flags & MDB_RAWPART) && ftruncate(env->me_fd, env->me_mapsize) < 0)
 			return ErrCode();
 	}
@@ -5957,7 +5993,7 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 		if (rc)
 			return ErrCode();
 		flags &= ~MDB_RAWPART;
-		if (S_ISBLK(st.st_mode))
+		if (S_ISBLK(st.st_mode) || S_ISCHR(st.st_mode))
 			flags |= MDB_RAWPART | MDB_NOSUBDIR;
 	}
 #endif

From de08119a296a4e8569fa04d7bfcbc7b3c2b41f2f Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 6 Aug 2018 13:09:37 +0100
Subject: [PATCH 388/504] Add mdb_env_set_pagesize()

---
 libraries/liblmdb/lmdb.h  | 10 ++++++++++
 libraries/liblmdb/mdb.c   | 15 +++++++++++++++
 libraries/liblmdb/mtest.c |  1 +
 3 files changed, 26 insertions(+)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 67b6667127..d996f38bdf 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -908,6 +908,16 @@ int  mdb_env_get_fd(MDB_env *env, mdb_filehandle_t *fd);
 	 */
 int  mdb_env_set_mapsize(MDB_env *env, mdb_size_t size);
 
+
+	/** @brief Set the size of DB pages in bytes.
+	 *
+	 * The size defaults to the OS page size. Smaller or larger values may be
+	 * desired depending on the size of keys and values being used. Also, an
+	 * explicit size may need to be set when using filesystems like ZFS which
+	 * don't use the OS page size.
+	 */
+int  mdb_env_set_pagesize(MDB_env *env, int size);
+
 	/** @brief Set the maximum number of threads/reader slots for the environment.
 	 *
 	 * This defines the number of slots in the lock table that is used to track readers in the
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 5b22841242..68d1d212a5 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -4439,6 +4439,8 @@ mdb_env_read_header(MDB_env *env, int prev, MDB_meta *meta)
 		env->me_mapsize &= ~(VM_ALIGN-1);
 		env->me_psize = env->me_os_psize;
 		rc = mdb_env_map(env, NULL);
+		if (rc)
+			return rc;
 		p = (MDB_page *)env->me_map;
 		for (i=0; i<NUM_METAS; i++) {
 			if (!F_ISSET(p->mp_flags, P_META))
@@ -11270,6 +11272,19 @@ mdb_env_get_fd(MDB_env *env, mdb_filehandle_t *arg)
 	return MDB_SUCCESS;
 }
 
+int ESECT
+mdb_env_set_pagesize(MDB_env *env, int size)
+{
+	if (!env || env->me_map)
+		return EINVAL;
+	if (size > MAX_PAGESIZE || size < 256)
+		return EINVAL;
+	if (size & (size-1))
+		return EINVAL;
+	env->me_os_psize = size;
+	return MDB_SUCCESS;
+}
+
 /** Common code for #mdb_stat() and #mdb_env_stat().
  * @param[in] env the environment to operate in.
  * @param[in] db the #MDB_db record containing the stats to return.
diff --git a/libraries/liblmdb/mtest.c b/libraries/liblmdb/mtest.c
index c1c9abb8f7..925f2d1ba7 100644
--- a/libraries/liblmdb/mtest.c
+++ b/libraries/liblmdb/mtest.c
@@ -47,6 +47,7 @@ int main(int argc,char * argv[])
 		E(mdb_env_create(&env));
 		E(mdb_env_set_maxreaders(env, 1));
 		E(mdb_env_set_mapsize(env, 10485760));
+		E(mdb_env_set_pagesize(env, 1024));
 		E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP /*|MDB_NOSYNC*/, 0664));
 
 		E(mdb_txn_begin(env, NULL, 0, &txn));

From 1738a2a7799239620b70fba156f918895c554c4b Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sat, 10 Oct 2020 16:29:01 +0100
Subject: [PATCH 389/504] Fix fallthru warning

---
 libraries/liblmdb/mdb_dump.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb_dump.c b/libraries/liblmdb/mdb_dump.c
index e0cca9ad25..0016caa8fb 100644
--- a/libraries/liblmdb/mdb_dump.c
+++ b/libraries/liblmdb/mdb_dump.c
@@ -189,7 +189,7 @@ int main(int argc, char *argv[])
 			break;
 		case 'l':
 			list = 1;
-			/*FALLTHROUGH*/;
+			/*FALLTHROUGH*/
 		case 'a':
 			if (subname)
 				usage(prog);

From 1002664c3317833dbc2c7ad4fb7b9421abed2706 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sun, 11 Oct 2020 01:32:30 +0100
Subject: [PATCH 390/504] Fix rawpart flag collision

---
 libraries/liblmdb/mdb.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 68d1d212a5..99c70075f2 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1578,14 +1578,14 @@ struct MDB_env {
 #endif /* _WIN32 */
 	/** Failed to update the meta page. Probably an I/O error. */
 #define	MDB_FATAL_ERROR	0x80000000U
+	/** using a raw block device */
+#define	MDB_RAWPART		0x40000000U
 	/** Some fields are initialized. */
 #define	MDB_ENV_ACTIVE	0x20000000U
 	/** me_txkey is set */
 #define	MDB_ENV_TXKEY	0x10000000U
 	/** fdatasync is unreliable */
 #define	MDB_FSYNCONLY	0x08000000U
-	/** using a raw block device */
-#define	MDB_RAWPART		0x04000000U
 	uint32_t 	me_flags;		/**< @ref mdb_env */
 	unsigned int	me_psize;	/**< DB page size, inited from me_os_psize */
 	unsigned int	me_os_psize;	/**< OS page size, from #GET_PAGESIZE */

From d1814f7e5d1dfca709ddbf9090c2ab636d32d3b5 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sat, 10 Oct 2020 23:32:56 +0100
Subject: [PATCH 391/504] ITS#9017 fixes for encryption

---
 libraries/liblmdb/mdb.c | 95 ++++++++++++++++++++++-------------------
 1 file changed, 51 insertions(+), 44 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 99c70075f2..58b13e154f 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1446,6 +1446,7 @@ struct MDB_txn {
 #define MDB_TXN_DIRTY		0x04		/**< must write, even if dirty list is empty */
 #define MDB_TXN_SPILLS		0x08		/**< txn or a parent has spilled pages */
 #define MDB_TXN_HAS_CHILD	0x10		/**< txn has an #MDB_txn.%mt_child */
+#define MDB_TXN_DIRTYNUM	0x20		/**< dirty list uses nump list */
 	/** most operations on the txn are currently illegal */
 #define MDB_TXN_BLOCKED		(MDB_TXN_FINISHED|MDB_TXN_ERROR|MDB_TXN_HAS_CHILD)
 /** @} */
@@ -1620,6 +1621,7 @@ struct MDB_env {
 	 *	Unused except for a dummy element when #MDB_WRITEMAP.
 	 */
 	MDB_ID2L	me_dirty_list;
+	int			*me_dirty_nump;
 	/** Max number of freelist items that can fit in a single overflow page */
 	int			me_maxfree_1pg;
 	/** Max size of a node on a page */
@@ -1630,8 +1632,8 @@ struct MDB_env {
 	int		me_live_reader;		/**< have liveness lock in reader table */
 #ifdef _WIN32
 	int		me_pidquery;		/**< Used in OpenProcess */
-	OVERLAPPED	*ov;			/**< Used for for overlapping I/O requests */
-	int		ovs;				/**< Count of OVERLAPPEDs */
+	OVERLAPPED		*me_ov;			/**< Used for overlapping I/O requests */
+	int		me_ovs;				/**< Count of MDB_overlaps */
 #endif
 #ifdef MDB_USE_POSIX_MUTEX	/* Posix mutexes reside in shared mem */
 #	define		me_rmutex	me_txns->mti_rmutex /**< Shared reader lock */
@@ -2202,8 +2204,16 @@ mdb_dlist_free(MDB_txn *txn)
 	MDB_ID2L dl = txn->mt_u.dirty_list;
 	unsigned i, n = dl[0].mid;
 
-	for (i = 1; i <= n; i++) {
-		mdb_dpage_free(env, dl[i].mptr);
+	if (txn->mt_flags & MDB_TXN_DIRTYNUM) {
+		int *dl_nump = env->me_dirty_nump;
+		for (i = 1; i <= n; i++) {
+			mdb_dpage_free_n(env, dl[i].mptr, dl_nump[i]);
+		}
+		txn->mt_flags ^= MDB_TXN_DIRTYNUM;
+	} else {
+		for (i = 1; i <= n; i++) {
+			mdb_dpage_free(env, dl[i].mptr);
+		}
 	}
 	dl[0].mid = 0;
 }
@@ -3912,18 +3922,17 @@ mdb_page_flush(MDB_txn *txn, int keep)
 	MDB_env		*env = txn->mt_env;
 	MDB_ID2L	dl = txn->mt_u.dirty_list;
 	unsigned	psize = env->me_psize, j;
-	int			i, pagecount = dl[0].mid, rc;
+	int			i, pagecount = dl[0].mid, rc, *dl_nump, nump = 1;
 	size_t		size = 0;
 	MDB_OFF_T	pos = 0;
 	pgno_t		pgno = 0;
 	MDB_page	*dp = NULL;
 #if MDB_RPAGE_CACHE
-	int			nump = 1;
 	MDB_page	*encp;
 #endif
 #ifdef _WIN32
-	OVERLAPPED	ov;
-	MDB_page    *wdp;
+	OVERLAPPED	*ov, *this_ov;
+	MDB_page	*wdp;
 	int async_i = 0;
 	HANDLE fd = (env->me_flags & MDB_NOSYNC) ? env->me_fd : env->me_ovfd;
 #else
@@ -3946,25 +3955,35 @@ mdb_page_flush(MDB_txn *txn, int keep)
 		goto done;
 	}
 
+	/* setup nump list, flag that it's in use */
+	dl_nump = env->me_dirty_nump;
+	for (n=1; n<=pagecount; n++) {
+		dp = dl[n].mptr;
+		dl_nump[n] = IS_OVERFLOW(dp) ? dp->mp_pages : 1;
+	}
+	n = 0;
+	txn->mt_flags |= MDB_TXN_DIRTYNUM;
+
 #ifdef _WIN32
-	if (pagecount - keep >= env->ovs) {
+	if (pagecount - keep >= env->me_ovs) {
 		/* ran out of room in ov array, and re-malloc, copy handles and free previous */
 		int ovs = (pagecount - keep) * 1.5; /* provide extra padding to reduce number of re-allocations */
 		int new_size = ovs * sizeof(OVERLAPPED);
 		ov = malloc(new_size);
 		if (ov == NULL)
 			return ENOMEM;
-		int previous_size = env->ovs * sizeof(OVERLAPPED);
-		memcpy(ov, env->ov, previous_size); /* Copy previous OVERLAPPED data to retain event handles */
+		int previous_size = env->me_ovs * sizeof(OVERLAPPED);
+		memcpy(ov, env->me_ov, previous_size); /* Copy previous OVERLAPPED data to retain event handles */
 		/* And clear rest of memory */
-		memset(&ov[env->ovs], 0, new_size - previous_size);
-		if (env->ovs > 0) {
-			free(env->ov); /* release previous allocation */
+		memset(&ov[env->me_ovs], 0, new_size - previous_size);
+		if (env->me_ovs > 0) {
+			free(env->me_ov); /* release previous allocation */
 		}
 
-		env->ov = ov;
-		env->ovs = ovs;
+		env->me_ov = ov;
+		env->me_ovs = ovs;
 	}
+	ov = env->me_ov;
 #endif
 
 	/* Write the pages */
@@ -3982,15 +4001,8 @@ mdb_page_flush(MDB_txn *txn, int keep)
 			dp->mp_txnid = txn->mt_txnid;
 			pos = pgno * psize;
 			size = psize;
-#if MDB_RPAGE_CACHE
-			nump = 1;
-			if (IS_OVERFLOW(dp)) {
-				nump = dp->mp_pages;
-				size *= dp->mp_pages;
-			}
-#else
-			if (IS_OVERFLOW(dp)) size *= dp->mp_pages;
-#endif
+			nump = dl_nump[i];
+			size *= nump;
 		}
 		/* Write up to MDB_COMMIT_PAGES dirty pages at a time. */
 		if (pos!=next_pos || n==MDB_COMMIT_PAGES || wsize+size>MAX_WRITE
@@ -4013,7 +4025,7 @@ retry_write:
 				rc = 0;
 				/* Write previous page(s) */
 #ifdef _WIN32
-				OVERLAPPED *this_ov = &ov[async_i];
+				this_ov = &ov[async_i];
 				/* Clear status, and keep hEvent, we reuse that */
 				this_ov->Internal = 0;
 				this_ov->Offset = wpos & 0xffffffff;
@@ -4033,6 +4045,7 @@ retry_write:
 						DPRINTF(("WriteFile: %d", rc));
 						return rc;
 					}
+					rc = 0;
 				}
 				async_i++;
 #else /* _WIN32 */
@@ -4067,15 +4080,6 @@ retry_seek:
 					}
 				}
 #endif /* _WIN32 */
-#if MDB_RPAGE_CACHE
-				if (env->me_encfunc) {
-					int j, num1;
-					for (j=0; j<n; j++) {
-						num1 = 1 + (iov[j].iov_len > psize);
-						mdb_dpage_free_n(env, (MDB_page *)iov[j].iov_base, num1);
-					}
-				}
-#endif
 				if (rc)
 					return rc;
 				n = 0;
@@ -4096,7 +4100,9 @@ retry_seek:
 			out.mv_size = size;
 			out.mv_data = encp;
 			env->me_encfunc(&in, &out, env->me_enckey, 1);
+			mdb_dpage_free_n(env, dp, nump);
 			dp = encp;
+			dl[i].mptr = dp;
 		}
 #endif
 #ifdef _WIN32
@@ -4143,9 +4149,6 @@ retry_seek:
 
     if (!(env->me_flags & MDB_WRITEMAP)) {
         /* Don't free pages when using writemap (can only get here in NOSYNC mode in Windows)
-         * MIPS has cache coherency issues, this is a no-op everywhere else
-         * Note: for any size >= on-chip cache size, entire on-chip cache is
-         * flushed.
          */
 		for (i = keep; ++i <= pagecount; ) {
 			dp = dl[i].mptr;
@@ -4155,9 +4158,10 @@ retry_seek:
 				dl[j].mid = dp->mp_pgno;
 				continue;
 			}
-			mdb_dpage_free(env, dp);
+			mdb_dpage_free_n(env, dp, dl_nump[i]);
 		}
 	}
+	txn->mt_flags ^= MDB_TXN_DIRTYNUM;
 
 done:
 	i--;
@@ -5205,7 +5209,7 @@ mdb_env_open2(MDB_env *env, int prev)
 		if (!NtCreateSection)
 			return MDB_PROBLEM;
 	}
-	env->ovs = 0;
+	env->me_ovs = 0;
 #endif /* _WIN32 */
 
 #ifdef BROKEN_FDATASYNC
@@ -6010,6 +6014,8 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 		if (!((env->me_free_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX)) &&
 			  (env->me_dirty_list = calloc(dl_size, sizeof(MDB_ID2)))))
 			rc = ENOMEM;
+		if (env->me_dirty_list && !(env->me_dirty_nump = malloc(dl_size * sizeof(int))))
+			rc = ENOMEM;
 	}
 
 	env->me_flags = flags;
@@ -6142,6 +6148,7 @@ mdb_env_close_active(MDB_env *env, int excl)
 	free(env->me_dbflags);
 	free(env->me_path);
 	free(env->me_dirty_list);
+	free(env->me_dirty_nump);
 #if MDB_RPAGE_CACHE
 	if (MDB_REMAPPING(env->me_flags)) {
 	if (env->me_txn0 && env->me_txn0->mt_rpages)
@@ -6180,11 +6187,11 @@ mdb_env_close_active(MDB_env *env, int excl)
 	if (env->me_mfd != INVALID_HANDLE_VALUE)
 		(void) close(env->me_mfd);
 #ifdef _WIN32
-	if (env->ovs > 0) {
-		for (i = 0; i < env->ovs; i++) {
-			CloseHandle(env->ov[i].hEvent);
+	if (env->me_ovs > 0) {
+		for (i = 0; i < env->me_ovs; i++) {
+			CloseHandle(env->me_ov[i].hEvent);
 		}
-		free(env->ov);
+		free(env->me_ov);
 	}
 	if (env->me_ovfd != INVALID_HANDLE_VALUE)
 		(void) close(env->me_ovfd);

From 8dc526c54fb64c0f65e7f72e500c936d8374e1b0 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sun, 11 Oct 2020 13:56:52 +0100
Subject: [PATCH 392/504] ITS#9364 rework crypto API

And add support for per-page checksums. Reserve space for checksum
at tail of page. Pass pgno+txnid as IV input for encryption.
---
 libraries/liblmdb/lmdb.h      |  43 ++++--
 libraries/liblmdb/mdb.c       | 250 ++++++++++++++++++++++++++--------
 libraries/liblmdb/mtest_enc.c |  14 +-
 3 files changed, 232 insertions(+), 75 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index d996f38bdf..2c23b33bed 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -317,14 +317,26 @@ typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *rel
  *
  * Encrypt or decrypt the data in src and store the result in dst using the
  * provided key. The result must be the same number of bytes as the input.
- * The input size will always be a multiple of the page size.
  * @param[in] src The input data to be transformed.
  * @param[out] dst Storage for the result.
- * @param[in] key An array of two values: key[0] is the encryption key,
- * and key[1] is the initialization vector.
+ * @param[in] key An array of three values: key[0] is the encryption key,
+ * key[1] is the initialization vector, and key[2] is the authentication
+ * data, if any.
  * @param[in] encdec 1 to encrypt, 0 to decrypt.
+ * @return A non-zero error value on failure and 0 on success.
  */
-typedef void (MDB_enc_func)(const MDB_val *src, MDB_val *dst, const MDB_val *key, int encdec);
+typedef int (MDB_enc_func)(const MDB_val *src, MDB_val *dst, const MDB_val *key, int encdec);
+
+/** @brief A callback function used to checksum pages in the env.
+ *
+ * Compute the checksum of the data in src and store the result in dst,
+ * An optional key may be used with keyed hash algorithms.
+ * @param[in] src The input data to be transformed.
+ * @param[out] dst Storage for the result.
+ * @param[in] key An encryption key, if encryption was configured. This
+ * parameter will be NULL if there is no key.
+ */
+typedef void (MDB_sum_func)(const MDB_val *src, MDB_val *dst, const MDB_val *key);
 #endif
 
 /** @defgroup	mdb_env	Environment Flags
@@ -506,8 +518,12 @@ typedef enum MDB_cursor_op {
 #define MDB_BAD_DBI		(-30780)
 	/** Unexpected problem - txn should abort */
 #define MDB_PROBLEM		(-30779)
+	/** Page checksum incorrect */
+#define MDB_BAD_CHECKSUM	(-30778)
+	/** Encryption/decryption failed */
+#define MDB_CRYPTO_FAIL		(-30777)
 	/** The last defined error code */
-#define MDB_LAST_ERRCODE	MDB_PROBLEM
+#define MDB_LAST_ERRCODE	MDB_CRYPTO_FAIL
 /** @} */
 
 /** @brief Statistics for a database in the environment */
@@ -1017,11 +1033,22 @@ int  mdb_env_set_assert(MDB_env *env, MDB_assert_func *func);
 	 * It implicitly sets #MDB_REMAP_CHUNKS on the env.
 	 * @param[in] env An environment handle returned by #mdb_env_create().
 	 * @param[in] func An #MDB_enc_func function.
-	 * @param[in] key An array of two values: key[0] is the encryption key,
-	 * and key[1] is the initialization vector.
+	 * @param[in] key The encryption key.
+	 * @param[in] size The size of authentication data in bytes, if any.
+	 * Set this to zero for unauthenticated encryption mechanisms.
 	 * @return A non-zero error value on failure and 0 on success.
 	 */
-int mdb_env_set_encrypt(MDB_env *env, MDB_enc_func *func, const MDB_val *key);
+int mdb_env_set_encrypt(MDB_env *env, MDB_enc_func *func, const MDB_val *key, unsigned int size);
+
+	/** @brief Set checksums on an environment.
+	 *
+	 * This must be called before #mdb_env_open().
+	 * @param[in] env An environment handle returned by #mdb_env_create().
+	 * @param[in] func An #MDB_sum_func function.
+	 * @param[in] size The size of computed checksum values, in bytes.
+	 * @return A non-zero error value on failure and 0 on success.
+	 */
+int mdb_env_set_checksum(MDB_env *env, MDB_sum_func *func, unsigned int size);
 #endif
 
 	/** @brief Create a transaction for use with the environment.
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 58b13e154f..96210f54ce 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1048,7 +1048,7 @@ typedef struct MDB_page {
 #define METADATA(p)	 ((void *)((char *)(p) + PAGEHDRSZ))
 
 	/** ITS#7713, change PAGEBASE to handle 65536 byte pages */
-#define	PAGEBASE	((MDB_DEVEL) ? PAGEHDRSZ : 0)
+#define	PAGEBASE	PAGEHDRSZ
 
 	/** Number of nodes on a page */
 #define NUMKEYS(p)	 (((p)->mp_lower - (PAGEHDRSZ-PAGEBASE)) >> 1)
@@ -1649,11 +1649,15 @@ struct MDB_env {
 #if MDB_RPAGE_CACHE
 	MDB_ID3L	me_rpages;	/**< like #mt_rpages, but global to env */
 	pthread_mutex_t	me_rpmutex;	/**< control access to #me_rpages */
+	MDB_sum_func *me_sumfunc;	/**< checksum env data */
+	unsigned short me_sumsize;	/**< size of per-page checksums */
 #define MDB_ERPAGE_SIZE	16384
 #define MDB_ERPAGE_MAX	(MDB_ERPAGE_SIZE-1)
+	unsigned short me_esumsize;	/**< size of per-page authentication data */
 	unsigned int me_rpcheck;
+
 	MDB_enc_func *me_encfunc;	/**< encrypt env data */
-	MDB_val		me_enckey[2];	/**< key and IV for env encryption */
+	MDB_val		me_enckey;	/**< key for env encryption */
 #endif
 	void		*me_userctx;	 /**< User-settable context */
 	MDB_assert_func *me_assert_func; /**< Callback for assertion failures */
@@ -3911,6 +3915,13 @@ mdb_freelist_save(MDB_txn *txn)
 	return rc;
 }
 
+#if MDB_RPAGE_CACHE
+static int mdb_rpage_decrypt(MDB_env *env, MDB_ID3 *id3, int rem, int numpgs);
+static int mdb_page_encrypt(MDB_env *env, MDB_page *in, MDB_page *out, size_t size);
+static int mdb_page_chk_checksum(MDB_env *env, MDB_page *mp, size_t size);
+static void mdb_page_set_checksum(MDB_env *env, MDB_page *mp, size_t size);
+#endif
+
 /** Flush (some) dirty pages to the map, after clearing their dirty flag.
  * @param[in] txn the transaction that's being committed
  * @param[in] keep number of initial pages in dirty_list to keep dirty.
@@ -3927,9 +3938,6 @@ mdb_page_flush(MDB_txn *txn, int keep)
 	MDB_OFF_T	pos = 0;
 	pgno_t		pgno = 0;
 	MDB_page	*dp = NULL;
-#if MDB_RPAGE_CACHE
-	MDB_page	*encp;
-#endif
 #ifdef _WIN32
 	OVERLAPPED	*ov, *this_ov;
 	MDB_page	*wdp;
@@ -4090,16 +4098,17 @@ retry_seek:
 			wsize = 0;
 		}
 #if MDB_RPAGE_CACHE
+		if (env->me_sumfunc) {
+			mdb_page_set_checksum(env, dp, size);
+		}
 		if (env->me_encfunc) {
-			MDB_val in, out;
-			encp = mdb_page_malloc(txn, nump, 0);
+			MDB_page *encp = mdb_page_malloc(txn, nump, 0);
 			if (!encp)
 				return ENOMEM;
-			in.mv_size = size;
-			in.mv_data = dp;
-			out.mv_size = size;
-			out.mv_data = encp;
-			env->me_encfunc(&in, &out, env->me_enckey, 1);
+			if (mdb_page_encrypt(env, dp, encp, size)) {
+				mdb_dpage_free_n(env, encp, nump);
+				return MDB_CRYPTO_FAIL;
+			}
 			mdb_dpage_free_n(env, dp, nump);
 			dp = encp;
 			dl[i].mptr = dp;
@@ -4580,12 +4589,11 @@ mdb_env_init_meta(MDB_env *env, MDB_meta *meta)
 	*(MDB_meta *)METADATA(q) = *meta;
 
 #if MDB_RPAGE_CACHE
-	if ((env->me_flags & MDB_ENCRYPT) && env->me_enckey[1].mv_size) {
-		/* save the IV in tail of page 0 */
+	if (env->me_sumsize) {
+		/* save the checksum size in tail of page 0 */
 		char *ptr = (char *)q;
 		unsigned short *u = (unsigned short *)(ptr-2);
-		*u = env->me_enckey[1].mv_size;
-		memcpy(ptr - 2 - env->me_enckey[1].mv_size, env->me_enckey[1].mv_data, env->me_enckey[1].mv_size);
+		*u = env->me_sumsize;
 	}
 #endif
 	DO_PWRITE(rc, env->me_fd, p, psize * NUM_METAS, len, 0);
@@ -5331,17 +5339,12 @@ mdb_env_open2(MDB_env *env, int prev)
 		return MDB_INCOMPATIBLE;
 
 #if MDB_RPAGE_CACHE
-	if (!newenv && env->me_flags & MDB_ENCRYPT) {
-		/* for encrypted env, read IV from tail of page 0 */
-		char *ptr = env->me_map + env->me_psize, *ekey;
+	if (!newenv && env->me_sumfunc) {
+		/* for checksums, check sum size from tail of page 0 */
+		char *ptr = env->me_map + env->me_psize;
 		unsigned short *u = (unsigned short *)(ptr - 2);
-		env->me_enckey[1].mv_size = *u;
-		ekey = realloc(env->me_enckey[0].mv_data, env->me_enckey[0].mv_size + env->me_enckey[1].mv_size);
-		if (!ekey)
-			return ENOMEM;
-		env->me_enckey[0].mv_data = ekey;
-		env->me_enckey[1].mv_data = ekey + env->me_enckey[0].mv_size;
-		memcpy(env->me_enckey[1].mv_data, ptr - 2 - env->me_enckey[1].mv_size, env->me_enckey[1].mv_size);
+		if (*u != env->me_sumsize)
+			return MDB_BAD_CHECKSUM;
 	}
 #endif
 
@@ -5912,11 +5915,9 @@ mdb_env_envflags(MDB_env *env)
 #if MDB_RPAGE_CACHE
 			if (!env->me_encfunc) {
 				static mdb_size_t k = (MDB_SIZE_MAX/67*73) | 1;
-				mdb_size_t iv = ((mdb_size_t)env ^ env->me_pid) * k;
-				MDB_val keys[2] = { {sizeof(k), &k}, {sizeof(iv), NULL} };
+				MDB_val key = {sizeof(k), &k};
 				int rc;
-				keys[1].mv_data = &iv;
-				rc = mdb_env_set_encrypt(env, mdb_enctest, keys);
+				rc = mdb_env_set_encrypt(env, mdb_enctest, &key, 0);
 				if (rc)
 					return rc;
 			}
@@ -6300,7 +6301,7 @@ mdb_env_close(MDB_env *env)
 
 	mdb_env_close_active(env, 0);
 #if MDB_RPAGE_CACHE
-	free(env->me_enckey[0].mv_data);
+	free(env->me_enckey.mv_data);
 #endif
 	free(env);
 }
@@ -6551,7 +6552,35 @@ mdb_cursor_push(MDB_cursor *mc, MDB_page *mp)
 }
 
 #if MDB_RPAGE_CACHE
-static void mdb_rpage_decrypt(MDB_env *env, MDB_ID3 *id3, int rem, int numpgs);
+
+static int
+mdb_rpage_encsum(MDB_env *env, MDB_ID3 *id3, unsigned rem, int numpgs)
+{
+	int rc = 0;
+	if (env->me_encfunc) {
+		unsigned short muse = id3->muse;
+		rc = mdb_rpage_decrypt(env, id3, rem, numpgs);
+		if (!rc && env->me_sumfunc && muse != id3->muse) {
+			MDB_page *p = (MDB_page *)(id3->menc + rem * env->me_psize);
+			rc = mdb_page_chk_checksum(env, p, numpgs * env->me_psize);
+		}
+	} else {
+		if (!(id3->muse & (1 << rem))) {
+			MDB_page *p;
+			int bit;
+			/* If this is an overflow page, set all use bits to the end */
+			if (rem + numpgs > MDB_RPAGE_CHUNK)
+				bit = 0xffff;
+			else
+				bit = 1;
+
+			id3->muse |= (bit << rem);
+			p = (MDB_page *)(id3->mptr + rem * env->me_psize);
+			rc = mdb_page_chk_checksum(env, p, numpgs * env->me_psize);
+		}
+	}
+	return rc;
+}
 
 /** Map a read-only page.
  * There are two levels of tracking in use, a per-txn list and a per-env list.
@@ -6704,8 +6733,9 @@ mdb_rpage_get(MDB_txn *txn, pgno_t pg0, int numpgs, MDB_page **ret)
 		id3.menc = tl[x].menc;
 		id3.muse = tl[x].muse;
 		tl[x].mref++;
-		if (env->me_encfunc) {
-			mdb_rpage_decrypt(env, &id3, rem, numpgs);
+		if (env->me_encfunc || env->me_sumfunc) {
+			rc = mdb_rpage_encsum(env, &id3, rem, numpgs);
+			if (rc) return rc;
 			tl[x].muse = id3.muse;
 		}
 		goto ok;
@@ -6799,8 +6829,10 @@ retry:
 					el[x].muse = id3.muse;
 				} else {
 					id3.mid = pg0;
-					if (env->me_encfunc) {
-						mdb_rpage_decrypt(env, &id3, rem, numpgs);
+					if (env->me_encfunc || env->me_sumfunc) {
+						rc = mdb_rpage_encsum(env, &id3, rem, numpgs);
+						if (rc)
+							goto fail;
 						el[x].muse = id3.muse;
 					}
 					pthread_mutex_unlock(&env->me_rpmutex);
@@ -6808,8 +6840,10 @@ retry:
 				}
 			}
 			el[x].mref++;
-			if (env->me_encfunc) {
-				mdb_rpage_decrypt(env, &id3, rem, numpgs);
+			if (env->me_encfunc || env->me_sumfunc) {
+				rc = mdb_rpage_encsum(env, &id3, rem, numpgs);
+				if (rc)
+					goto fail;
 				el[x].muse = id3.muse;
 			}
 			pthread_mutex_unlock(&env->me_rpmutex);
@@ -6862,7 +6896,11 @@ fail:
 				rc = ENOMEM;
 				goto fail;
 			}
-			mdb_rpage_decrypt(env, &id3, rem, numpgs);
+		}
+		if (env->me_encfunc || env->me_sumfunc) {
+			rc = mdb_rpage_encsum(env, &id3, rem, numpgs);
+			if (rc)
+				goto fail;
 		}
 		mdb_mid3l_insert(el, &id3);
 		pthread_mutex_unlock(&env->me_rpmutex);
@@ -6874,22 +6912,50 @@ found:
 ok:
 	base = (char *)(env->me_encfunc ? id3.menc : id3.mptr);
 	p = (MDB_page *)(base + rem * env->me_psize);
-	if (env->me_encfunc)
-		mdb_rpage_decrypt(env, &id3, rem, numpgs);
+	rc = MDB_SUCCESS;
+	if (env->me_encfunc || env->me_sumfunc) {
+		rc = mdb_rpage_encsum(env, &id3, rem, numpgs);
+	}
 #if MDB_DEBUG	/* we don't need this check any more */
 	if (IS_OVERFLOW(p)) {
 		mdb_tassert(txn, p->mp_pages + rem <= id3.mcnt);
 	}
 #endif
 	*ret = p;
-	return MDB_SUCCESS;
+	return rc;
 }
 
-static void mdb_rpage_decrypt(MDB_env *env, MDB_ID3 *id3, int rem, int numpgs)
+static int mdb_page_encrypt(MDB_env *env, MDB_page *dp, MDB_page *encp, size_t size)
 {
+	MDB_val in, out, enckeys[3];
+	int xsize = sizeof(pgno_t) + sizeof(txnid_t);
+	in.mv_size = size - xsize;
+	in.mv_data = (char *)dp + xsize;
+	if (env->me_esumsize) {
+		in.mv_size -= env->me_esumsize;
+		enckeys[2].mv_size = env->me_esumsize;
+		enckeys[2].mv_data = in.mv_data + in.mv_size;
+	} else {
+		enckeys[2].mv_size = 0;
+		enckeys[2].mv_data = 0;
+	}
+	out.mv_size = in.mv_size;
+	out.mv_data = (char *)encp + xsize;
+	encp->mp_pgno = dp->mp_pgno;
+	encp->mp_txnid = dp->mp_txnid;
+	enckeys[0] = env->me_enckey;
+	enckeys[1].mv_size = xsize;
+	enckeys[1].mv_data = dp;
+	return env->me_encfunc(&in, &out, enckeys, 1);
+}
+
+static int mdb_rpage_decrypt(MDB_env *env, MDB_ID3 *id3, int rem, int numpgs)
+{
+	int rc = 0;
 	if (!(id3->muse & (1 << rem))) {
-		MDB_val in, out;
+		MDB_val in, out, enckeys[3];
 		int bit;
+		int xsize = sizeof(pgno_t) + sizeof(txnid_t);
 
 		/* If this is an overflow page, set all use bits to the end */
 		if (rem + numpgs > MDB_RPAGE_CHUNK)
@@ -6898,12 +6964,32 @@ static void mdb_rpage_decrypt(MDB_env *env, MDB_ID3 *id3, int rem, int numpgs)
 			bit = 1;
 
 		id3->muse |= (bit << rem);
-		in.mv_size = numpgs * env->me_psize;
-		in.mv_data = (char *)id3->mptr + rem * env->me_psize;
+		in.mv_size = numpgs * env->me_psize - xsize;
+		in.mv_data = (char *)id3->mptr + rem * env->me_psize + xsize;
+		enckeys[0] = env->me_enckey;
+		enckeys[1].mv_size = xsize;
+		enckeys[1].mv_data = in.mv_data - xsize;
+		if (env->me_esumsize) {
+			in.mv_size -= env->me_esumsize;
+			enckeys[2].mv_size = env->me_esumsize;
+			enckeys[2].mv_data = in.mv_data + in.mv_size;
+		} else {
+			enckeys[2].mv_size = 0;
+			enckeys[2].mv_data = 0;
+		}
 		out.mv_size = in.mv_size;
-		out.mv_data = (char *)id3->menc + rem * env->me_psize;
-		env->me_encfunc(&in, &out, env->me_enckey, 0);
+		out.mv_data = (char *)id3->menc + rem * env->me_psize + xsize;
+		if (env->me_encfunc(&in, &out, enckeys, 0))
+			rc = MDB_CRYPTO_FAIL;
+		else {
+			MDB_page *penc, *pclr;
+			penc = (MDB_page *)enckeys[1].mv_data;
+			pclr = (MDB_page *)(out.mv_data - xsize);
+			pclr->mp_pgno = penc->mp_pgno;
+			pclr->mp_txnid = penc->mp_txnid;
+		}
 	}
+	return rc;
 }
 
 /** zero out decrypted pages before freeing them */
@@ -6923,6 +7009,38 @@ static void mdb_rpage_dispose(MDB_env *env, MDB_ID3 *id3)
 	}
 	free(id3->menc);
 }
+
+static void mdb_page_set_checksum(MDB_env *env, MDB_page *mp, size_t size)
+{
+	MDB_val src, dst, *key;
+	src.mv_size = size - env->me_sumsize;
+	src.mv_data = mp;
+	dst.mv_size = env->me_sumsize;
+	dst.mv_data = src.mv_data + src.mv_size;
+	if (env->me_encfunc)
+		key = &env->me_enckey;
+	else
+		key = NULL;
+	env->me_sumfunc(&src, &dst, key);
+}
+
+static int mdb_page_chk_checksum(MDB_env *env, MDB_page *mp, size_t size)
+{
+	MDB_val src, dst, chk, *key;
+	char sumbuf[256];
+	src.mv_size = size - env->me_sumsize;
+	src.mv_data = mp;
+	chk.mv_size = env->me_sumsize;
+	chk.mv_data = src.mv_data + src.mv_size;
+	dst.mv_size = env->me_sumsize;
+	dst.mv_data = sumbuf;
+	if (env->me_encfunc)
+		key = &env->me_enckey;
+	else
+		key = NULL;
+	env->me_sumfunc(&src, &dst, key);
+	return memcmp(chk.mv_data, dst.mv_data, env->me_sumsize) ? MDB_BAD_CHECKSUM : 0;
+}
 #endif
 
 /** Find the address of the page corresponding to a given page number.
@@ -8759,6 +8877,10 @@ mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp)
 	np->mp_flags |= flags;
 	np->mp_lower = (PAGEHDRSZ-PAGEBASE);
 	np->mp_upper = mc->mc_txn->mt_env->me_psize - PAGEBASE;
+#if MDB_RPAGE_CACHE
+	np->mp_upper -= mc->mc_txn->mt_env->me_sumsize;
+	np->mp_upper -= mc->mc_txn->mt_env->me_esumsize;
+#endif
 
 	if (IS_BRANCH(np))
 		mc->mc_db->md_branch_pages++;
@@ -11234,29 +11356,39 @@ mdb_env_set_assert(MDB_env *env, MDB_assert_func *func)
 
 #if MDB_RPAGE_CACHE
 int ESECT
-mdb_env_set_encrypt(MDB_env *env, MDB_enc_func *func, const MDB_val *key)
+mdb_env_set_encrypt(MDB_env *env, MDB_enc_func *func, const MDB_val *key, unsigned int size)
 {
-	char *kdata, *ivdata;
+	char *kdata;
 
 	if (!env || !func || !key)
 		return EINVAL;
 	if (env->me_flags & MDB_ENV_ACTIVE)
 		return EINVAL;
-	if (! (kdata = malloc(key[0].mv_size + key[1].mv_size)))
+	if (! (kdata = malloc(key[0].mv_size)))
 		return ENOMEM;
 
-	ivdata = kdata + key[0].mv_size;
-	memcpy(kdata,  key[0].mv_data, key[0].mv_size);
-	memcpy(ivdata, key[1].mv_data, key[1].mv_size);
-	free(env->me_enckey[0].mv_data);
-	env->me_enckey[0].mv_data = kdata;
-	env->me_enckey[0].mv_size = key[0].mv_size;
-	env->me_enckey[1].mv_data = ivdata;
-	env->me_enckey[1].mv_size = key[1].mv_size;
+	memcpy(kdata,  key->mv_data, key->mv_size);
+	free(env->me_enckey.mv_data);
+	env->me_enckey.mv_data = kdata;
+	env->me_enckey.mv_size = key->mv_size;
 	env->me_encfunc = func;
+	if (size)
+		env->me_esumsize = size;
 	env->me_flags |= MDB_REMAP_CHUNKS | MDB_ENCRYPT;
 	return MDB_SUCCESS;
 }
+
+int ESECT
+mdb_env_set_checksum(MDB_env *env, MDB_sum_func *func, unsigned int size)
+{
+	if (!env || !func || !size)
+		return EINVAL;
+	if (env->me_flags & MDB_ENV_ACTIVE)
+		return EINVAL;
+	env->me_sumfunc = func;
+	env->me_sumsize = size;
+	return MDB_SUCCESS;
+}
 #endif
 
 int ESECT
diff --git a/libraries/liblmdb/mtest_enc.c b/libraries/liblmdb/mtest_enc.c
index ecb0875730..266d5417b1 100644
--- a/libraries/liblmdb/mtest_enc.c
+++ b/libraries/liblmdb/mtest_enc.c
@@ -21,9 +21,10 @@
 #define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \
 	"%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort()))
 
-static void encfunc(const MDB_val *src, MDB_val *dst, const MDB_val *key, int encdec)
+static int encfunc(const MDB_val *src, MDB_val *dst, const MDB_val *key, int encdec)
 {
 	chacha8(src->mv_data, src->mv_size, key[0].mv_data, key[1].mv_data, dst->mv_data);
+	return 0;
 }
 
 int main(int argc,char * argv[])
@@ -36,11 +37,10 @@ int main(int argc,char * argv[])
 	MDB_stat mst;
 	MDB_cursor *cursor, *cur2;
 	MDB_cursor_op op;
-	MDB_val enckey[2];
+	MDB_val enckey;
 	int count;
 	int *values;
 	char sval[32] = "";
-	char eiv[] = {3, 1, 4, 1, 5, 9, 2, 6};
 	char ekey[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
 		17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32};
 
@@ -53,15 +53,13 @@ int main(int argc,char * argv[])
 			values[i] = rand()%1024;
 	    }
     
-		enckey[0].mv_data = ekey;
-		enckey[0].mv_size = sizeof(ekey);
-		enckey[1].mv_data = eiv;
-		enckey[1].mv_size = sizeof(eiv);
+		enckey.mv_data = ekey;
+		enckey.mv_size = sizeof(ekey);
 
 		E(mdb_env_create(&env));
 		E(mdb_env_set_maxreaders(env, 1));
 		E(mdb_env_set_mapsize(env, 10485760));
-		E(mdb_env_set_encrypt(env, encfunc, enckey));
+		E(mdb_env_set_encrypt(env, encfunc, &enckey, 0));
 		E(mdb_env_open(env, "./testdb", 0 /*|MDB_NOSYNC*/, 0664));
 
 		E(mdb_txn_begin(env, NULL, 0, &txn));

From 0bc8a4e9294469459c03a18cdad906979e6b6c20 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sun, 11 Oct 2020 14:07:26 +0100
Subject: [PATCH 393/504] ITS#9364 add error code texts

---
 libraries/liblmdb/mdb.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 96210f54ce..80f1762ace 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1828,6 +1828,8 @@ static char *const mdb_errstr[] = {
 	"MDB_BAD_VALSIZE: Unsupported size of key/DB name/data, or wrong DUPFIXED size",
 	"MDB_BAD_DBI: The specified DBI handle was closed/changed unexpectedly",
 	"MDB_PROBLEM: Unexpected problem - txn should abort",
+	"MDB_BAD_CHECKSUM: Page checksum mismatch",
+	"MDB_CRYPTO_FAIL: Page encryption or decryption failed",
 };
 
 char *

From 93c72a77269249cf3f9ef4b9e1b85212e2d2e4b2 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sun, 11 Oct 2020 17:29:54 +0100
Subject: [PATCH 394/504] ITS#9364 add sample for authenticated encryption

---
 libraries/liblmdb/Makefile     |   5 +-
 libraries/liblmdb/crypto.c     |  62 +++++++++++
 libraries/liblmdb/lmdb.h       |  29 ++++-
 libraries/liblmdb/mdb.c        |  10 +-
 libraries/liblmdb/mtest_enc2.c | 188 +++++++++++++++++++++++++++++++++
 5 files changed, 288 insertions(+), 6 deletions(-)
 create mode 100644 libraries/liblmdb/crypto.c
 create mode 100644 libraries/liblmdb/mtest_enc2.c

diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile
index 292b0fa592..b8c03cf238 100644
--- a/libraries/liblmdb/Makefile
+++ b/libraries/liblmdb/Makefile
@@ -42,7 +42,7 @@ ILIBS	= liblmdb.a liblmdb$(SOEXT)
 IPROGS	= mdb_stat mdb_copy mdb_dump mdb_load mdb_drop
 IDOCS	= mdb_stat.1 mdb_copy.1 mdb_dump.1 mdb_load.1 mdb_drop.1
 PROGS	= $(IPROGS) mtest mtest2 mtest3 mtest4 mtest5
-RPROGS	= mtest_remap mtest_enc
+RPROGS	= mtest_remap mtest_enc mtest_enc2
 
 all:	$(ILIBS) $(PROGS)
 # Requires CPPFLAGS=-DMDB_VL32 and/or -DMDB_RPAGE_CACHE
@@ -85,6 +85,9 @@ mtest5:	mtest5.o liblmdb.a
 mtest6:	mtest6.o liblmdb.a
 mtest_remap:  mtest_remap.o liblmdb.a
 mtest_enc:    mtest_enc.o chacha8.o liblmdb.a
+mtest_enc2:	  mtest_enc2.o crypto.o liblmdb.a
+	$(CC) $(LDFLAGS) -pthread -o $@ $^ -lcrypto
+
 
 mdb.o: mdb.c lmdb.h midl.h
 	$(CC) $(CFLAGS) $(CPPFLAGS) -c mdb.c
diff --git a/libraries/liblmdb/crypto.c b/libraries/liblmdb/crypto.c
new file mode 100644
index 0000000000..db7620e668
--- /dev/null
+++ b/libraries/liblmdb/crypto.c
@@ -0,0 +1,62 @@
+#include <string.h>
+
+#include <openssl/engine.h>
+
+#include "lmdb.h"
+
+MDB_crypto_hooks MDB_crypto;
+
+static EVP_CIPHER *cipher;
+
+static int str2key(const char *passwd, MDB_val *key)
+{
+	unsigned int size;
+	EVP_MD_CTX *mdctx = EVP_MD_CTX_new();
+	EVP_DigestInit_ex(mdctx, EVP_sha256(), NULL);
+	EVP_DigestUpdate(mdctx, "Just a Constant", sizeof("Just a Constant"));
+	EVP_DigestUpdate(mdctx, passwd, strlen(passwd));
+	EVP_DigestFinal_ex(mdctx, key->mv_data, &size);
+	EVP_MD_CTX_free(mdctx);
+	return 0;
+}
+
+static int encfunc(const MDB_val *src, MDB_val *dst, const MDB_val *key, int encdec)
+{
+	unsigned char iv[12];
+	int ivl, outl, rc;
+	mdb_size_t *ptr;
+	EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new();
+
+	ptr = key[1].mv_data;
+	ivl = ptr[0] & 0xffffffff;
+	memcpy(iv, &ivl, 4);
+	memcpy(iv+4, ptr+1, sizeof(mdb_size_t));
+	EVP_CipherInit_ex(ctx, cipher, NULL, key[0].mv_data, iv, encdec);
+	EVP_CIPHER_CTX_set_padding(ctx, 0);
+	if (!encdec) {
+		EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_TAG, key[2].mv_size, key[2].mv_data);
+	}
+	rc = EVP_CipherUpdate(ctx, dst->mv_data, &outl, src->mv_data, src->mv_size);
+	if (rc)
+		rc = EVP_CipherFinal_ex(ctx, key[2].mv_data, &outl);
+	if (rc && encdec) {
+		EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_GET_TAG, key[2].mv_size, key[2].mv_data);
+	}
+	EVP_CIPHER_CTX_free(ctx);
+	return rc == 0;
+}
+
+static const MDB_crypto_funcs table = {
+	str2key,
+	encfunc,
+	NULL,
+	32,
+	16,
+	0
+};
+
+MDB_crypto_funcs *MDB_crypto()
+{
+	cipher = (EVP_CIPHER *)EVP_chacha20_poly1305();
+	return (MDB_crypto_funcs *)&table;
+}
diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 2c23b33bed..2fba3a5a8d 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -522,8 +522,10 @@ typedef enum MDB_cursor_op {
 #define MDB_BAD_CHECKSUM	(-30778)
 	/** Encryption/decryption failed */
 #define MDB_CRYPTO_FAIL		(-30777)
+	/** Environment encryption mismatch */
+#define MDB_ENV_ENCRYPTION	(-30776)
 	/** The last defined error code */
-#define MDB_LAST_ERRCODE	MDB_CRYPTO_FAIL
+#define MDB_LAST_ERRCODE	MDB_ENV_ENCRYPTION
 /** @} */
 
 /** @brief Statistics for a database in the environment */
@@ -1723,6 +1725,31 @@ int	mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx);
 int	mdb_reader_check(MDB_env *env, int *dead);
 /**	@} */
 
+/** @defgroup crypto LMDB Encryption Helper API
+ *	@{
+ *	@brief Helpers for setting up encryption
+ */
+
+	/** @brief A function for converting a string into an encryption key.
+	 *
+	 * @param[in] passwd The string to be converted.
+	 * @param[in,out] key The resulting key. The caller must
+	 * provide the space for the key.
+	 * @return 0 on success, non-zero on failure.
+	 */
+typedef int (MDB_str2key_func)(const char *passwd, MDB_val *key);
+
+typedef struct MDB_crypto_funcs {
+	MDB_str2key_func *mcf_str2key;
+	MDB_enc_func *mcf_encfunc;
+	MDB_sum_func *mcf_sumfunc;
+	int mcf_keysize;
+	int mcf_esumsize;
+	int mcf_sumsize;
+} MDB_crypto_funcs;
+
+typedef MDB_crypto_funcs *(MDB_crypto_hooks)();
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 80f1762ace..b82c1447c4 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1830,6 +1830,7 @@ static char *const mdb_errstr[] = {
 	"MDB_PROBLEM: Unexpected problem - txn should abort",
 	"MDB_BAD_CHECKSUM: Page checksum mismatch",
 	"MDB_CRYPTO_FAIL: Page encryption or decryption failed",
+	"MDB_ENV_ENCRYPTION: Environment encryption mismatch",
 };
 
 char *
@@ -5338,7 +5339,7 @@ mdb_env_open2(MDB_env *env, int prev)
 		}
 	}
 	if ((env->me_flags ^ env->me_metas[0]->mm_flags) & MDB_ENCRYPT)
-		return MDB_INCOMPATIBLE;
+		return MDB_ENV_ENCRYPTION;
 
 #if MDB_RPAGE_CACHE
 	if (!newenv && env->me_sumfunc) {
@@ -6933,16 +6934,17 @@ static int mdb_page_encrypt(MDB_env *env, MDB_page *dp, MDB_page *encp, size_t s
 	int xsize = sizeof(pgno_t) + sizeof(txnid_t);
 	in.mv_size = size - xsize;
 	in.mv_data = (char *)dp + xsize;
+	out.mv_size = in.mv_size;
+	out.mv_data = (char *)encp + xsize;
 	if (env->me_esumsize) {
 		in.mv_size -= env->me_esumsize;
+		out.mv_size -= env->me_esumsize;
 		enckeys[2].mv_size = env->me_esumsize;
-		enckeys[2].mv_data = in.mv_data + in.mv_size;
+		enckeys[2].mv_data = out.mv_data + out.mv_size;
 	} else {
 		enckeys[2].mv_size = 0;
 		enckeys[2].mv_data = 0;
 	}
-	out.mv_size = in.mv_size;
-	out.mv_data = (char *)encp + xsize;
 	encp->mp_pgno = dp->mp_pgno;
 	encp->mp_txnid = dp->mp_txnid;
 	enckeys[0] = env->me_enckey;
diff --git a/libraries/liblmdb/mtest_enc2.c b/libraries/liblmdb/mtest_enc2.c
new file mode 100644
index 0000000000..c99663d15b
--- /dev/null
+++ b/libraries/liblmdb/mtest_enc2.c
@@ -0,0 +1,188 @@
+/* mtest_enc.c - memory-mapped database tester/toy with encryption */
+/*
+ * Copyright 2011-2017 Howard Chu, Symas Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the Symas
+ * Dual-Use License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * source distribution.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+#include "lmdb.h"
+
+#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr)
+#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0))
+#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \
+	"%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort()))
+
+extern MDB_crypto_hooks MDB_crypto;
+MDB_crypto_funcs *cf;
+
+int main(int argc,char * argv[])
+{
+	int i = 0, j = 0, rc;
+	MDB_env *env;
+	MDB_dbi dbi;
+	MDB_val key, data;
+	MDB_txn *txn;
+	MDB_stat mst;
+	MDB_cursor *cursor, *cur2;
+	MDB_cursor_op op;
+	MDB_val enckey;
+	int count;
+	int *values;
+	char sval[32] = "";
+	char password[] = "This is my passphrase for now";
+	char ekey[32];
+
+	srand(time(NULL));
+
+	    count = (rand()%384) + 64;
+	    values = (int *)malloc(count*sizeof(int));
+
+	    for(i = 0;i<count;i++) {
+			values[i] = rand()%1024;
+	    }
+    
+		cf = MDB_crypto();
+		enckey.mv_data = ekey;
+		enckey.mv_size = sizeof(ekey);
+		cf->mcf_str2key(password, &enckey);
+
+		E(mdb_env_create(&env));
+		E(mdb_env_set_maxreaders(env, 1));
+		E(mdb_env_set_mapsize(env, 10485760));
+		E(mdb_env_set_encrypt(env, cf->mcf_encfunc, &enckey, cf->mcf_esumsize));
+		E(mdb_env_open(env, "./testdb", 0 /*|MDB_NOSYNC*/, 0664));
+
+		E(mdb_txn_begin(env, NULL, 0, &txn));
+		E(mdb_dbi_open(txn, NULL, 0, &dbi));
+   
+		key.mv_size = sizeof(int);
+		key.mv_data = sval;
+
+		printf("Adding %d values\n", count);
+	    for (i=0;i<count;i++) {	
+			sprintf(sval, "%03x %d foo bar", values[i], values[i]);
+			/* Set <data> in each iteration, since MDB_NOOVERWRITE may modify it */
+			data.mv_size = sizeof(sval);
+			data.mv_data = sval;
+			if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) {
+				j++;
+				data.mv_size = sizeof(sval);
+				data.mv_data = sval;
+			}
+	    }
+		if (j) printf("%d duplicates skipped\n", j);
+		E(mdb_txn_commit(txn));
+		E(mdb_env_stat(env, &mst));
+
+		E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
+		E(mdb_cursor_open(txn, dbi, &cursor));
+		while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
+			printf("key: %p %.*s, data: %p %.*s\n",
+				key.mv_data,  (int) key.mv_size,  (char *) key.mv_data,
+				data.mv_data, (int) data.mv_size, (char *) data.mv_data);
+		}
+		CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
+		mdb_cursor_close(cursor);
+		mdb_txn_abort(txn);
+
+		j=0;
+		key.mv_data = sval;
+	    for (i= count - 1; i > -1; i-= (rand()%5)) {
+			j++;
+			txn=NULL;
+			E(mdb_txn_begin(env, NULL, 0, &txn));
+			sprintf(sval, "%03x ", values[i]);
+			if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) {
+				j--;
+				mdb_txn_abort(txn);
+			} else {
+				E(mdb_txn_commit(txn));
+			}
+	    }
+	    free(values);
+		printf("Deleted %d values\n", j);
+
+		E(mdb_env_stat(env, &mst));
+		E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
+		E(mdb_cursor_open(txn, dbi, &cursor));
+		printf("Cursor next\n");
+		while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
+			printf("key: %.*s, data: %.*s\n",
+				(int) key.mv_size,  (char *) key.mv_data,
+				(int) data.mv_size, (char *) data.mv_data);
+		}
+		CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
+		printf("Cursor last\n");
+		E(mdb_cursor_get(cursor, &key, &data, MDB_LAST));
+		printf("key: %.*s, data: %.*s\n",
+			(int) key.mv_size,  (char *) key.mv_data,
+			(int) data.mv_size, (char *) data.mv_data);
+		printf("Cursor prev\n");
+		while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) {
+			printf("key: %.*s, data: %.*s\n",
+				(int) key.mv_size,  (char *) key.mv_data,
+				(int) data.mv_size, (char *) data.mv_data);
+		}
+		CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
+		printf("Cursor last/prev\n");
+		E(mdb_cursor_get(cursor, &key, &data, MDB_LAST));
+			printf("key: %.*s, data: %.*s\n",
+				(int) key.mv_size,  (char *) key.mv_data,
+				(int) data.mv_size, (char *) data.mv_data);
+		E(mdb_cursor_get(cursor, &key, &data, MDB_PREV));
+			printf("key: %.*s, data: %.*s\n",
+				(int) key.mv_size,  (char *) key.mv_data,
+				(int) data.mv_size, (char *) data.mv_data);
+
+		mdb_cursor_close(cursor);
+		mdb_txn_abort(txn);
+
+		printf("Deleting with cursor\n");
+		E(mdb_txn_begin(env, NULL, 0, &txn));
+		E(mdb_cursor_open(txn, dbi, &cur2));
+		for (i=0; i<50; i++) {
+			if (RES(MDB_NOTFOUND, mdb_cursor_get(cur2, &key, &data, MDB_NEXT)))
+				break;
+			printf("key: %p %.*s, data: %p %.*s\n",
+				key.mv_data,  (int) key.mv_size,  (char *) key.mv_data,
+				data.mv_data, (int) data.mv_size, (char *) data.mv_data);
+			E(mdb_del(txn, dbi, &key, NULL));
+		}
+
+		printf("Restarting cursor in txn\n");
+		for (op=MDB_FIRST, i=0; i<=32; op=MDB_NEXT, i++) {
+			if (RES(MDB_NOTFOUND, mdb_cursor_get(cur2, &key, &data, op)))
+				break;
+			printf("key: %p %.*s, data: %p %.*s\n",
+				key.mv_data,  (int) key.mv_size,  (char *) key.mv_data,
+				data.mv_data, (int) data.mv_size, (char *) data.mv_data);
+		}
+		mdb_cursor_close(cur2);
+		E(mdb_txn_commit(txn));
+
+		printf("Restarting cursor outside txn\n");
+		E(mdb_txn_begin(env, NULL, 0, &txn));
+		E(mdb_cursor_open(txn, dbi, &cursor));
+		for (op=MDB_FIRST, i=0; i<=32; op=MDB_NEXT, i++) {
+			if (RES(MDB_NOTFOUND, mdb_cursor_get(cursor, &key, &data, op)))
+				break;
+			printf("key: %p %.*s, data: %p %.*s\n",
+				key.mv_data,  (int) key.mv_size,  (char *) key.mv_data,
+				data.mv_data, (int) data.mv_size, (char *) data.mv_data);
+		}
+		mdb_cursor_close(cursor);
+		mdb_txn_abort(txn);
+
+		mdb_dbi_close(env, dbi);
+		mdb_env_close(env);
+
+	return 0;
+}

From 34fd28154395117bd062d17f8a003df22d56b5cf Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sun, 11 Oct 2020 18:17:07 +0100
Subject: [PATCH 395/504] ITS#9364 tweak sample

Cheat with OpenSSL 1.1 internal structures to avoid malloc/free per page
---
 libraries/liblmdb/crypto.c | 63 +++++++++++++++++++++++++++++++++-----
 1 file changed, 55 insertions(+), 8 deletions(-)

diff --git a/libraries/liblmdb/crypto.c b/libraries/liblmdb/crypto.c
index db7620e668..57461c7bf0 100644
--- a/libraries/liblmdb/crypto.c
+++ b/libraries/liblmdb/crypto.c
@@ -20,29 +20,76 @@ static int str2key(const char *passwd, MDB_val *key)
 	return 0;
 }
 
+/* cheats - internal OpenSSL 1.1 structures */
+typedef struct evp_cipher_ctx_st {
+    const EVP_CIPHER *cipher;
+    ENGINE *engine;             /* functional reference if 'cipher' is
+                                 * ENGINE-provided */
+    int encrypt;                /* encrypt or decrypt */
+    int buf_len;                /* number we have left */
+    unsigned char oiv[EVP_MAX_IV_LENGTH]; /* original iv */
+    unsigned char iv[EVP_MAX_IV_LENGTH]; /* working iv */
+    unsigned char buf[EVP_MAX_BLOCK_LENGTH]; /* saved partial block */
+    int num;                    /* used by cfb/ofb/ctr mode */
+    /* FIXME: Should this even exist? It appears unused */
+    void *app_data;             /* application stuff */
+    int key_len;                /* May change for variable length cipher */
+    unsigned long flags;        /* Various flags */
+    void *cipher_data;          /* per EVP data */
+    int final_used;
+    int block_mask;
+    unsigned char final[EVP_MAX_BLOCK_LENGTH]; /* possible final block */
+} EVP_CIPHER_CTX;
+
+#define	CHACHA_KEY_SIZE	32
+#define CHACHA_CTR_SIZE	16
+#define CHACHA_BLK_SIZE	64
+#define POLY1305_BLOCK_SIZE	16
+
+typedef struct {
+    union {
+        double align;   /* this ensures even sizeof(EVP_CHACHA_KEY)%8==0 */
+        unsigned int d[CHACHA_KEY_SIZE / 4];
+    } key;
+    unsigned int  counter[CHACHA_CTR_SIZE / 4];
+    unsigned char buf[CHACHA_BLK_SIZE];
+    unsigned int  partial_len;
+} EVP_CHACHA_KEY;
+
+typedef struct {
+    EVP_CHACHA_KEY key;
+    unsigned int nonce[12/4];
+    unsigned char tag[POLY1305_BLOCK_SIZE];
+    unsigned char tls_aad[POLY1305_BLOCK_SIZE];
+    struct { uint64_t aad, text; } len;
+    int aad, mac_inited, tag_len, nonce_len;
+    size_t tls_payload_length;
+} EVP_CHACHA_AEAD_CTX;
+
 static int encfunc(const MDB_val *src, MDB_val *dst, const MDB_val *key, int encdec)
 {
 	unsigned char iv[12];
 	int ivl, outl, rc;
 	mdb_size_t *ptr;
-	EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new();
+	EVP_CIPHER_CTX ctx = {0};
+	EVP_CHACHA_AEAD_CTX cactx = {0};
 
+	ctx.cipher_data = &cactx;
 	ptr = key[1].mv_data;
 	ivl = ptr[0] & 0xffffffff;
 	memcpy(iv, &ivl, 4);
 	memcpy(iv+4, ptr+1, sizeof(mdb_size_t));
-	EVP_CipherInit_ex(ctx, cipher, NULL, key[0].mv_data, iv, encdec);
-	EVP_CIPHER_CTX_set_padding(ctx, 0);
+	EVP_CipherInit_ex(&ctx, cipher, NULL, key[0].mv_data, iv, encdec);
+	EVP_CIPHER_CTX_set_padding(&ctx, 0);
 	if (!encdec) {
-		EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_TAG, key[2].mv_size, key[2].mv_data);
+		EVP_CIPHER_CTX_ctrl(&ctx, EVP_CTRL_AEAD_SET_TAG, key[2].mv_size, key[2].mv_data);
 	}
-	rc = EVP_CipherUpdate(ctx, dst->mv_data, &outl, src->mv_data, src->mv_size);
+	rc = EVP_CipherUpdate(&ctx, dst->mv_data, &outl, src->mv_data, src->mv_size);
 	if (rc)
-		rc = EVP_CipherFinal_ex(ctx, key[2].mv_data, &outl);
+		rc = EVP_CipherFinal_ex(&ctx, key[2].mv_data, &outl);
 	if (rc && encdec) {
-		EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_GET_TAG, key[2].mv_size, key[2].mv_data);
+		EVP_CIPHER_CTX_ctrl(&ctx, EVP_CTRL_AEAD_GET_TAG, key[2].mv_size, key[2].mv_data);
 	}
-	EVP_CIPHER_CTX_free(ctx);
 	return rc == 0;
 }
 

From 21d21a09b0af779b2f5c23d35f11bf7b38b8640f Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sun, 11 Oct 2020 18:19:10 +0100
Subject: [PATCH 396/504] ITS#9364 tweak crypto sample again

---
 libraries/liblmdb/crypto.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/crypto.c b/libraries/liblmdb/crypto.c
index 57461c7bf0..1cfa8b7fd9 100644
--- a/libraries/liblmdb/crypto.c
+++ b/libraries/liblmdb/crypto.c
@@ -72,7 +72,7 @@ static int encfunc(const MDB_val *src, MDB_val *dst, const MDB_val *key, int enc
 	int ivl, outl, rc;
 	mdb_size_t *ptr;
 	EVP_CIPHER_CTX ctx = {0};
-	EVP_CHACHA_AEAD_CTX cactx = {0};
+	EVP_CHACHA_AEAD_CTX cactx;
 
 	ctx.cipher_data = &cactx;
 	ptr = key[1].mv_data;

From 12c63d299ccb4ee3556b4ab83830e6e531aa2892 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 12 Oct 2020 03:09:48 +0100
Subject: [PATCH 397/504] ITS#9364 use crypto table properly

---
 libraries/liblmdb/mtest_enc2.c | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/libraries/liblmdb/mtest_enc2.c b/libraries/liblmdb/mtest_enc2.c
index c99663d15b..e8e8b1aee2 100644
--- a/libraries/liblmdb/mtest_enc2.c
+++ b/libraries/liblmdb/mtest_enc2.c
@@ -11,6 +11,7 @@
  * source distribution.
  */
 #include <stdio.h>
+#include <string.h>
 #include <stdlib.h>
 #include <time.h>
 #include "lmdb.h"
@@ -37,8 +38,8 @@ int main(int argc,char * argv[])
 	int count;
 	int *values;
 	char sval[32] = "";
-	char password[] = "This is my passphrase for now";
-	char ekey[32];
+	char password[] = "This is my passphrase for now...";
+	char *ekey;
 
 	srand(time(NULL));
 
@@ -50,14 +51,23 @@ int main(int argc,char * argv[])
 	    }
     
 		cf = MDB_crypto();
-		enckey.mv_data = ekey;
-		enckey.mv_size = sizeof(ekey);
-		cf->mcf_str2key(password, &enckey);
 
 		E(mdb_env_create(&env));
 		E(mdb_env_set_maxreaders(env, 1));
 		E(mdb_env_set_mapsize(env, 10485760));
-		E(mdb_env_set_encrypt(env, cf->mcf_encfunc, &enckey, cf->mcf_esumsize));
+		if (cf->mcf_sumfunc) {
+			E(mdb_env_set_checksum(env, cf->mcf_sumfunc, cf->mcf_sumsize));
+		}
+		if (cf->mcf_encfunc) {
+			ekey = malloc(cf->mcf_keysize);
+			enckey.mv_data = ekey;
+			enckey.mv_size = cf->mcf_keysize;
+			if (cf->mcf_str2key)
+				cf->mcf_str2key(password, &enckey);
+			else
+				strncpy(ekey, password, cf->mcf_keysize);
+			E(mdb_env_set_encrypt(env, cf->mcf_encfunc, &enckey, cf->mcf_esumsize));
+		}
 		E(mdb_env_open(env, "./testdb", 0 /*|MDB_NOSYNC*/, 0664));
 
 		E(mdb_txn_begin(env, NULL, 0, &txn));

From b220a6655b269153f32b89ec113a73f983b5f8e3 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 12 Oct 2020 21:57:05 +0100
Subject: [PATCH 398/504] ITS#9364 Add crypto support to all tools

Using dynamically loaded crypto modules
---
 libraries/liblmdb/Makefile     |  23 +++++---
 libraries/liblmdb/crypto.c     |  28 ++++++---
 libraries/liblmdb/mdb_copy.c   |  24 +++++++-
 libraries/liblmdb/mdb_drop.c   |  23 +++++++-
 libraries/liblmdb/mdb_dump.c   |  21 ++++++-
 libraries/liblmdb/mdb_load.c   |  22 ++++++-
 libraries/liblmdb/mdb_stat.c   |  23 ++++++--
 libraries/liblmdb/module.c     | 101 +++++++++++++++++++++++++++++++++
 libraries/liblmdb/module.h     |  16 ++++++
 libraries/liblmdb/mtest_enc2.c |  11 +++-
 10 files changed, 264 insertions(+), 28 deletions(-)
 create mode 100644 libraries/liblmdb/module.c
 create mode 100644 libraries/liblmdb/module.h

diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile
index b8c03cf238..c252b50e21 100644
--- a/libraries/liblmdb/Makefile
+++ b/libraries/liblmdb/Makefile
@@ -24,9 +24,11 @@ W	= -W -Wall -Wno-unused-parameter -Wbad-function-cast -Wuninitialized
 THREADS = -pthread
 OPT = -O2 -g
 CFLAGS	= $(THREADS) $(OPT) $(W) $(XCFLAGS)
+LDFLAGS = $(THREADS)
 LDLIBS	= 
 SOLIBS	= 
 SOEXT	= .so
+LDL		= -ldl
 prefix	= /usr/local
 exec_prefix = $(prefix)
 bindir = $(exec_prefix)/bin
@@ -72,11 +74,16 @@ liblmdb$(SOEXT):	mdb.lo midl.lo
 #	$(CC) $(LDFLAGS) -pthread -shared -Wl,-Bsymbolic -o $@ mdb.o midl.o $(SOLIBS)
 	$(CC) $(LDFLAGS) -pthread -shared -o $@ mdb.lo midl.lo $(SOLIBS)
 
-mdb_stat: mdb_stat.o liblmdb.a
-mdb_copy: mdb_copy.o liblmdb.a
-mdb_dump: mdb_dump.o liblmdb.a
-mdb_load: mdb_load.o liblmdb.a
-mdb_drop: mdb_drop.o liblmdb.a
+mdb_stat: mdb_stat.o module.o liblmdb.a
+	$(CC) $(LDFLAGS) -o $@ $^ $(LDL)
+mdb_copy: mdb_copy.o module.o liblmdb.a
+	$(CC) $(LDFLAGS) -o $@ $^ $(LDL)
+mdb_dump: mdb_dump.o module.o liblmdb.a
+	$(CC) $(LDFLAGS) -o $@ $^ $(LDL)
+mdb_load: mdb_load.o module.o liblmdb.a
+	$(CC) $(LDFLAGS) -o $@ $^ $(LDL)
+mdb_drop: mdb_drop.o module.o liblmdb.a
+	$(CC) $(LDFLAGS) -o $@ $^ $(LDL)
 mtest:    mtest.o    liblmdb.a
 mtest2:	mtest2.o liblmdb.a
 mtest3:	mtest3.o liblmdb.a
@@ -85,9 +92,11 @@ mtest5:	mtest5.o liblmdb.a
 mtest6:	mtest6.o liblmdb.a
 mtest_remap:  mtest_remap.o liblmdb.a
 mtest_enc:    mtest_enc.o chacha8.o liblmdb.a
-mtest_enc2:	  mtest_enc2.o crypto.o liblmdb.a
-	$(CC) $(LDFLAGS) -pthread -o $@ $^ -lcrypto
+mtest_enc2:	  mtest_enc2.o module.o liblmdb.a crypto.lm
+	$(CC) $(LDFLAGS) -pthread -o $@ mtest_enc2.o module.o liblmdb.a $(LDL)
 
+crypto.lm:	crypto.c
+	$(CC) -shared -o $@ -lcrypto
 
 mdb.o: mdb.c lmdb.h midl.h
 	$(CC) $(CFLAGS) $(CPPFLAGS) -c mdb.c
diff --git a/libraries/liblmdb/crypto.c b/libraries/liblmdb/crypto.c
index 1cfa8b7fd9..40665cbacc 100644
--- a/libraries/liblmdb/crypto.c
+++ b/libraries/liblmdb/crypto.c
@@ -1,3 +1,15 @@
+/* crypto.c - LMDB encryption helper module */
+/*
+ * Copyright 2020 Howard Chu, Symas Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the Symas
+ * Dual-Use License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * source distribution.
+ */
 #include <string.h>
 
 #include <openssl/engine.h>
@@ -8,7 +20,7 @@ MDB_crypto_hooks MDB_crypto;
 
 static EVP_CIPHER *cipher;
 
-static int str2key(const char *passwd, MDB_val *key)
+static int mcf_str2key(const char *passwd, MDB_val *key)
 {
 	unsigned int size;
 	EVP_MD_CTX *mdctx = EVP_MD_CTX_new();
@@ -66,7 +78,7 @@ typedef struct {
     size_t tls_payload_length;
 } EVP_CHACHA_AEAD_CTX;
 
-static int encfunc(const MDB_val *src, MDB_val *dst, const MDB_val *key, int encdec)
+static int mcf_encfunc(const MDB_val *src, MDB_val *dst, const MDB_val *key, int encdec)
 {
 	unsigned char iv[12];
 	int ivl, outl, rc;
@@ -93,17 +105,17 @@ static int encfunc(const MDB_val *src, MDB_val *dst, const MDB_val *key, int enc
 	return rc == 0;
 }
 
-static const MDB_crypto_funcs table = {
-	str2key,
-	encfunc,
+static const MDB_crypto_funcs mcf_table = {
+	mcf_str2key,
+	mcf_encfunc,
 	NULL,
-	32,
-	16,
+	CHACHA_KEY_SIZE,
+	POLY1305_BLOCK_SIZE,
 	0
 };
 
 MDB_crypto_funcs *MDB_crypto()
 {
 	cipher = (EVP_CIPHER *)EVP_chacha20_poly1305();
-	return (MDB_crypto_funcs *)&table;
+	return (MDB_crypto_funcs *)&mcf_table;
 }
diff --git a/libraries/liblmdb/mdb_copy.c b/libraries/liblmdb/mdb_copy.c
index cfcd13ac30..c0a006c73f 100644
--- a/libraries/liblmdb/mdb_copy.c
+++ b/libraries/liblmdb/mdb_copy.c
@@ -21,6 +21,7 @@
 #include <stdlib.h>
 #include <signal.h>
 #include "lmdb.h"
+#include "module.h"
 
 static void
 sighandle(int sig)
@@ -34,6 +35,9 @@ int main(int argc,char * argv[])
 	const char *progname = argv[0], *act;
 	unsigned flags = MDB_RDONLY;
 	unsigned cpflags = 0;
+	char *module = NULL, *password = NULL;
+	void *mlm = NULL;
+	char *errmsg;
 
 	for (; argc > 1 && argv[1][0] == '-'; argc--, argv++) {
 		if (argv[1][1] == 'n' && argv[1][2] == '\0')
@@ -45,15 +49,24 @@ int main(int argc,char * argv[])
 		else if (argv[1][1] == 'V' && argv[1][2] == '\0') {
 			printf("%s\n", MDB_VERSION_STRING);
 			exit(0);
+		} else if (argv[1][1] == 'm' && argv[1][2] == '\0') {
+			module = argv[2];
+			argc--;
+			argv++;
+		} else if (argv[1][1] == 'w' && argv[1][2] == '\0') {
+			password = argv[2];
+			argc--;
+			argv++;
 		} else
 			argc = 0;
 	}
 
 	if (argc<2 || argc>3) {
-		fprintf(stderr, "usage: %s [-V] [-c] [-n] [-v] srcpath [dstpath]\n", progname);
+		fprintf(stderr, "usage: %s [-V] [-c] [-n] [-v] [-m module [-w password]] srcpath [dstpath]\n", progname);
 		exit(EXIT_FAILURE);
 	}
 
+
 #ifdef SIGPIPE
 	signal(SIGPIPE, sighandle);
 #endif
@@ -66,6 +79,13 @@ int main(int argc,char * argv[])
 	act = "opening environment";
 	rc = mdb_env_create(&env);
 	if (rc == MDB_SUCCESS) {
+		if (module) {
+			mlm = mlm_setup(env, module, password, &errmsg);
+			if (!mlm) {
+				fprintf(stderr, "Failed to load crypto module: %s\n", errmsg);
+				exit(EXIT_FAILURE);
+			}
+		}
 		rc = mdb_env_open(env, argv[1], flags, 0600);
 	}
 	if (rc == MDB_SUCCESS) {
@@ -79,6 +99,8 @@ int main(int argc,char * argv[])
 		fprintf(stderr, "%s: %s failed, error %d (%s)\n",
 			progname, act, rc, mdb_strerror(rc));
 	mdb_env_close(env);
+	if (mlm)
+		mlm_unload(mlm);
 
 	return rc ? EXIT_FAILURE : EXIT_SUCCESS;
 }
diff --git a/libraries/liblmdb/mdb_drop.c b/libraries/liblmdb/mdb_drop.c
index 3d9d779b5b..3b31beb1be 100644
--- a/libraries/liblmdb/mdb_drop.c
+++ b/libraries/liblmdb/mdb_drop.c
@@ -19,6 +19,7 @@
 #include <unistd.h>
 #include <signal.h>
 #include "lmdb.h"
+#include "module.h"
 
 static volatile sig_atomic_t gotsig;
 
@@ -29,7 +30,7 @@ static void dumpsig( int sig )
 
 static void usage(char *prog)
 {
-	fprintf(stderr, "usage: %s [-V] [-n] [-d] [-s subdb] dbpath\n", prog);
+	fprintf(stderr, "usage: %s [-V] [-n] [-d] [-m module [-w password]] [-s subdb] dbpath\n", prog);
 	exit(EXIT_FAILURE);
 }
 
@@ -43,6 +44,9 @@ int main(int argc, char *argv[])
 	char *envname;
 	char *subname = NULL;
 	int envflags = 0, delete = 0;
+	char *module = NULL, *password = NULL;
+	void *mlm = NULL;
+	char *errmsg;
 
 	if (argc < 2) {
 		usage(prog);
@@ -54,7 +58,7 @@ int main(int argc, char *argv[])
 	 * -V: print version and exit
 	 * (default) empty the main DB
 	 */
-	while ((i = getopt(argc, argv, "dns:V")) != EOF) {
+	while ((i = getopt(argc, argv, "dm:ns:w:V")) != EOF) {
 		switch(i) {
 		case 'V':
 			printf("%s\n", MDB_VERSION_STRING);
@@ -69,6 +73,12 @@ int main(int argc, char *argv[])
 		case 's':
 			subname = optarg;
 			break;
+		case 'm':
+			module = optarg;
+			break;
+		case 'w':
+			password = optarg;
+			break;
 		default:
 			usage(prog);
 		}
@@ -92,6 +102,13 @@ int main(int argc, char *argv[])
 		fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc));
 		return EXIT_FAILURE;
 	}
+	if (module) {
+		mlm = mlm_setup(env, module, password, &errmsg);
+		if (!mlm) {
+			fprintf(stderr, "Failed to load crypto module: %s\n", errmsg);
+			goto env_close;
+		}
+	}
 
 	mdb_env_set_maxdbs(env, 2);
 
@@ -130,6 +147,8 @@ txn_abort:
 		mdb_txn_abort(txn);
 env_close:
 	mdb_env_close(env);
+	if (mlm)
+		mlm_unload(mlm);
 
 	return rc ? EXIT_FAILURE : EXIT_SUCCESS;
 }
diff --git a/libraries/liblmdb/mdb_dump.c b/libraries/liblmdb/mdb_dump.c
index 0016caa8fb..1b3d4dee83 100644
--- a/libraries/liblmdb/mdb_dump.c
+++ b/libraries/liblmdb/mdb_dump.c
@@ -19,6 +19,7 @@
 #include <unistd.h>
 #include <signal.h>
 #include "lmdb.h"
+#include "module.h"
 
 #define Yu	MDB_PRIy(u)
 
@@ -153,7 +154,7 @@ static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name)
 
 static void usage(char *prog)
 {
-	fprintf(stderr, "usage: %s [-V] [-f output] [-l] [-n] [-p] [-v] [-a|-s subdb] dbpath\n", prog);
+	fprintf(stderr, "usage: %s [-V] [-f output] [-l] [-n] [-p] [-v] [-m module [-w password]] [-a|-s subdb] dbpath\n", prog);
 	exit(EXIT_FAILURE);
 }
 
@@ -167,6 +168,8 @@ int main(int argc, char *argv[])
 	char *envname;
 	char *subname = NULL;
 	int alldbs = 0, envflags = 0, list = 0;
+	char *module = NULL, *password = NULL, *errmsg;
+	void *mlm = NULL;
 
 	if (argc < 2) {
 		usage(prog);
@@ -181,7 +184,7 @@ int main(int argc, char *argv[])
 	 * -V: print version and exit
 	 * (default) dump only the main DB
 	 */
-	while ((i = getopt(argc, argv, "af:lnps:vV")) != EOF) {
+	while ((i = getopt(argc, argv, "af:lm:nps:vw:V")) != EOF) {
 		switch(i) {
 		case 'V':
 			printf("%s\n", MDB_VERSION_STRING);
@@ -216,6 +219,12 @@ int main(int argc, char *argv[])
 				usage(prog);
 			subname = optarg;
 			break;
+		case 'm':
+			module = optarg;
+			break;
+		case 'w':
+			password = optarg;
+			break;
 		default:
 			usage(prog);
 		}
@@ -240,6 +249,14 @@ int main(int argc, char *argv[])
 		return EXIT_FAILURE;
 	}
 
+	if (module) {
+		mlm = mlm_setup(env, module, password, &errmsg);
+		if (!mlm) {
+			fprintf(stderr, "Failed to load crypto module: %s\n", errmsg);
+			goto env_close;
+		}
+	}
+
 	if (alldbs || subname) {
 		mdb_env_set_maxdbs(env, 2);
 	}
diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c
index 0a99e177f1..c9b2ad93f7 100644
--- a/libraries/liblmdb/mdb_load.c
+++ b/libraries/liblmdb/mdb_load.c
@@ -18,6 +18,7 @@
 #include <ctype.h>
 #include <unistd.h>
 #include "lmdb.h"
+#include "module.h"
 
 #define PRINT	1
 #define NOHDR	2
@@ -276,7 +277,7 @@ badend:
 
 static void usage(void)
 {
-	fprintf(stderr, "usage: %s [-V] [-a] [-f input] [-n] [-s name] [-N] [-T] dbpath\n", prog);
+	fprintf(stderr, "usage: %s [-V] [-a] [-f input] [-n] [-m module [-w password]] [-s name] [-N] [-T] dbpath\n", prog);
 	exit(EXIT_FAILURE);
 }
 
@@ -296,6 +297,8 @@ int main(int argc, char *argv[])
 	int envflags = MDB_NOSYNC, putflags = 0;
 	int dohdr = 0, append = 0;
 	MDB_val prevk;
+	char *module = NULL, *password = NULL, *errmsg;
+	void *mlm = NULL;
 
 	prog = argv[0];
 
@@ -311,7 +314,7 @@ int main(int argc, char *argv[])
 	 * -T: read plaintext
 	 * -V: print version and exit
 	 */
-	while ((i = getopt(argc, argv, "af:ns:NTV")) != EOF) {
+	while ((i = getopt(argc, argv, "af:m:ns:w:NTV")) != EOF) {
 		switch(i) {
 		case 'V':
 			printf("%s\n", MDB_VERSION_STRING);
@@ -339,6 +342,12 @@ int main(int argc, char *argv[])
 		case 'T':
 			mode |= NOHDR | PRINT;
 			break;
+		case 'm':
+			module = optarg;
+			break;
+		case 'w':
+			password = optarg;
+			break;
 		default:
 			usage();
 		}
@@ -359,6 +368,13 @@ int main(int argc, char *argv[])
 		fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc));
 		return EXIT_FAILURE;
 	}
+	if (module) {
+		mlm = mlm_setup(env, module, password, &errmsg);
+		if (!mlm) {
+			fprintf(stderr, "Failed to load crypto module: %s\n", errmsg);
+			goto env_close;
+		}
+	}
 
 	mdb_env_set_maxdbs(env, 2);
 
@@ -487,6 +503,8 @@ txn_abort:
 	mdb_txn_abort(txn);
 env_close:
 	mdb_env_close(env);
+	if (mlm)
+		mlm_unload(mlm);
 
 	return rc ? EXIT_FAILURE : EXIT_SUCCESS;
 }
diff --git a/libraries/liblmdb/mdb_stat.c b/libraries/liblmdb/mdb_stat.c
index ad1fef5052..1bb1bcb342 100644
--- a/libraries/liblmdb/mdb_stat.c
+++ b/libraries/liblmdb/mdb_stat.c
@@ -16,15 +16,14 @@
 #include <string.h>
 #include <unistd.h>
 #include "lmdb.h"
+#include "module.h"
 
 #define Z	MDB_FMT_Z
 #define Yu	MDB_PRIy(u)
 
 static void prstat(MDB_stat *ms)
 {
-#if 0
 	printf("  Page size: %u\n", ms->ms_psize);
-#endif
 	printf("  Tree depth: %u\n", ms->ms_depth);
 	printf("  Branch pages: %"Yu"\n",   ms->ms_branch_pages);
 	printf("  Leaf pages: %"Yu"\n",     ms->ms_leaf_pages);
@@ -34,7 +33,7 @@ static void prstat(MDB_stat *ms)
 
 static void usage(char *prog)
 {
-	fprintf(stderr, "usage: %s [-V] [-n] [-e] [-r[r]] [-f[f[f]]] [-v] [-a|-s subdb] dbpath\n", prog);
+	fprintf(stderr, "usage: %s [-V] [-n] [-e] [-r[r]] [-f[f[f]]] [-v] [-m module [-w password]] [-a|-s subdb] dbpath\n", prog);
 	exit(EXIT_FAILURE);
 }
 
@@ -50,6 +49,8 @@ int main(int argc, char *argv[])
 	char *envname;
 	char *subname = NULL;
 	int alldbs = 0, envinfo = 0, envflags = 0, freinfo = 0, rdrinfo = 0;
+	char *module = NULL, *password = NULL, *errmsg;
+	void *mlm = NULL;
 
 	if (argc < 2) {
 		usage(prog);
@@ -65,7 +66,7 @@ int main(int argc, char *argv[])
 	 * -V: print version and exit
 	 * (default) print stat of only the main DB
 	 */
-	while ((i = getopt(argc, argv, "Vaefnrs:v")) != EOF) {
+	while ((i = getopt(argc, argv, "Vaefm:nrs:vw:")) != EOF) {
 		switch(i) {
 		case 'V':
 			printf("%s\n", MDB_VERSION_STRING);
@@ -96,6 +97,12 @@ int main(int argc, char *argv[])
 				usage(prog);
 			subname = optarg;
 			break;
+		case 'm':
+			module = optarg;
+			break;
+		case 'w':
+			password = optarg;
+			break;
 		default:
 			usage(prog);
 		}
@@ -111,6 +118,14 @@ int main(int argc, char *argv[])
 		return EXIT_FAILURE;
 	}
 
+	if (module) {
+		mlm = mlm_setup(env, module, password, &errmsg);
+		if (!mlm) {
+			fprintf(stderr, "Failed to load crypto module: %s\n", errmsg);
+			goto env_close;
+		}
+	}
+
 	if (alldbs || subname) {
 		mdb_env_set_maxdbs(env, 4);
 	}
diff --git a/libraries/liblmdb/module.c b/libraries/liblmdb/module.c
new file mode 100644
index 0000000000..20e883beb8
--- /dev/null
+++ b/libraries/liblmdb/module.c
@@ -0,0 +1,101 @@
+/* module.c - helper for dynamically loading crypto module */
+/*
+ * Copyright 2020 Howard Chu, Symas Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the Symas
+ * Dual-Use License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * source distribution.
+ */
+#ifdef _WIN32
+#include <windows.h>
+#else
+#include <dlfcn.h>
+#endif
+
+#include <stddef.h>
+#include <string.h>
+
+#include "lmdb.h"
+#include "module.h"
+
+void *mlm_load(const char *file, const char *name, MDB_crypto_funcs **mcf_ptr, char **errmsg)
+{
+	MDB_crypto_hooks *hookfunc;
+	void *ret = NULL;
+	if (!name)
+		name = "MDB_crypto";
+
+#ifdef _WIN32
+	{
+		HINSTANCE mlm = LoadLibrary(file);
+		if (mlm) {
+			hookfunc = GetProcAddress(mlm, name);
+			if (hookfunc)
+				*mcf_ptr = hookfunc();
+			else {
+				*errmsg = "Crypto hook function not found";
+				FreeLibrary(mlm);
+				mlm = NULL;
+			}
+		} else {
+			*errmsg = GetLastError();
+		}
+		ret = (void *)mlm;
+	}
+#else
+	{
+		void *mlm = dlopen(file, RTLD_NOW);
+		if (mlm) {
+			hookfunc = dlsym(mlm, name);
+			if (hookfunc)
+				*mcf_ptr = hookfunc();
+			else {
+				*errmsg = "Crypto hook function not found";
+				dlclose(mlm);
+				mlm = NULL;
+			}
+		} else {
+			*errmsg = dlerror();
+		}
+		ret = mlm;
+	}
+#endif
+	return ret;
+}
+
+void mlm_unload(void *mlm)
+{
+#ifdef _WIN32
+	FreeLibrary((HINSTANCE)mlm);
+#else
+	dlclose(mlm);
+#endif
+}
+
+void *mlm_setup(MDB_env *env, const char *file, const char *password, char **errmsg)
+{
+	MDB_crypto_funcs *cf;
+	MDB_val enckey = {0};
+	void *mlm = mlm_load(file, NULL, &cf, errmsg);
+	if (mlm) {
+		if (cf->mcf_sumfunc) {
+			mdb_env_set_checksum(env, cf->mcf_sumfunc, cf->mcf_sumsize);
+		}
+		if (cf->mcf_encfunc && password) {
+			char keybuf[2048];
+			enckey.mv_data = keybuf;
+			enckey.mv_size = cf->mcf_keysize;
+			if (cf->mcf_str2key)
+				cf->mcf_str2key(password, &enckey);
+			else
+				strncpy(enckey.mv_data, password, enckey.mv_size);
+			mdb_env_set_encrypt(env, cf->mcf_encfunc, &enckey, cf->mcf_esumsize);
+			memset(enckey.mv_data, 0, enckey.mv_size);
+		}
+	}
+	return mlm;
+}
diff --git a/libraries/liblmdb/module.h b/libraries/liblmdb/module.h
new file mode 100644
index 0000000000..7c768bad11
--- /dev/null
+++ b/libraries/liblmdb/module.h
@@ -0,0 +1,16 @@
+/* module.h - helper for dynamically loading crypto module */
+/*
+ * Copyright 2020 Howard Chu, Symas Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the Symas
+ * Dual-Use License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * source distribution.
+ */
+
+void *mlm_load(const char *file, const char *name, MDB_crypto_funcs **mcf_ptr, char **errmsg);
+void mlm_unload(void *lm);
+void *mlm_setup(MDB_env *env, const char *file, const char *password, char **errmsg);
diff --git a/libraries/liblmdb/mtest_enc2.c b/libraries/liblmdb/mtest_enc2.c
index e8e8b1aee2..b69c935d79 100644
--- a/libraries/liblmdb/mtest_enc2.c
+++ b/libraries/liblmdb/mtest_enc2.c
@@ -15,13 +15,13 @@
 #include <stdlib.h>
 #include <time.h>
 #include "lmdb.h"
+#include "module.h"
 
 #define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr)
 #define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0))
 #define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \
 	"%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort()))
 
-extern MDB_crypto_hooks MDB_crypto;
 MDB_crypto_funcs *cf;
 
 int main(int argc,char * argv[])
@@ -40,6 +40,8 @@ int main(int argc,char * argv[])
 	char sval[32] = "";
 	char password[] = "This is my passphrase for now...";
 	char *ekey;
+	void *lm;
+	char *errmsg;
 
 	srand(time(NULL));
 
@@ -50,7 +52,11 @@ int main(int argc,char * argv[])
 			values[i] = rand()%1024;
 	    }
     
-		cf = MDB_crypto();
+		lm = lm_load("./crypto.lm", NULL, &cf, &errmsg);
+		if (!lm) {
+			fprintf(stderr,"Failed to load crypto module: %s\n", errmsg);
+			exit(1);
+		}
 
 		E(mdb_env_create(&env));
 		E(mdb_env_set_maxreaders(env, 1));
@@ -193,6 +199,7 @@ int main(int argc,char * argv[])
 
 		mdb_dbi_close(env, dbi);
 		mdb_env_close(env);
+		lm_unload(lm);
 
 	return 0;
 }

From 8e8371d125c9f4bccb208e2d80ee286d89c1e82e Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 12 Oct 2020 22:28:27 +0100
Subject: [PATCH 399/504] ITS#9364 Add docs for crypto modules

---
 libraries/liblmdb/lmdb.h       | 20 ++++++++++++++++----
 libraries/liblmdb/mdb_copy.1   | 13 +++++++++++++
 libraries/liblmdb/mdb_drop.1   | 13 +++++++++++++
 libraries/liblmdb/mdb_dump.1   | 13 +++++++++++++
 libraries/liblmdb/mdb_dump.c   |  2 ++
 libraries/liblmdb/mdb_load.1   | 13 +++++++++++++
 libraries/liblmdb/mdb_stat.1   | 13 +++++++++++++
 libraries/liblmdb/mdb_stat.c   |  2 ++
 libraries/liblmdb/mtest_enc2.c | 26 +++++---------------------
 9 files changed, 90 insertions(+), 25 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 2fba3a5a8d..128031ac34 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -1739,16 +1739,27 @@ int	mdb_reader_check(MDB_env *env, int *dead);
 	 */
 typedef int (MDB_str2key_func)(const char *passwd, MDB_val *key);
 
+	/** @brief A structure for dynamically loaded crypto modules.
+	 *
+	 * This is the information that the command line tools expect
+	 * in order to operate on encrypted or checksummed environments.
+	 */
 typedef struct MDB_crypto_funcs {
 	MDB_str2key_func *mcf_str2key;
 	MDB_enc_func *mcf_encfunc;
 	MDB_sum_func *mcf_sumfunc;
-	int mcf_keysize;
-	int mcf_esumsize;
-	int mcf_sumsize;
+	int mcf_keysize;	/**< The size of an encryption key, in bytes */
+	int mcf_esumsize;	/**< The size of the MAC, for authenticated encryption */
+	int mcf_sumsize;	/**< The size of the checksum, for plain checksums */
 } MDB_crypto_funcs;
 
-typedef MDB_crypto_funcs *(MDB_crypto_hooks)();
+	/** @brief The function that returns the #MDB_crypto_funcs structure.
+	 *
+	 * The command line tools expect this function to be named "MDB_crypto".
+	 * It must be exported by the dynamic module so that the tools can use it.
+	 * @return A pointer to a #MDB_crypto_funcs structure.
+	 */
+typedef MDB_crypto_funcs *(MDB_crypto_hooks)(void);
 
 #ifdef __cplusplus
 }
@@ -1756,6 +1767,7 @@ typedef MDB_crypto_funcs *(MDB_crypto_hooks)();
 /** @page tools LMDB Command Line Tools
 	The following describes the command line tools that are available for LMDB.
 	\li \ref mdb_copy_1
+	\li \ref mdb_drop_1
 	\li \ref mdb_dump_1
 	\li \ref mdb_load_1
 	\li \ref mdb_stat_1
diff --git a/libraries/liblmdb/mdb_copy.1 b/libraries/liblmdb/mdb_copy.1
index fdf266368d..762d3fbf57 100644
--- a/libraries/liblmdb/mdb_copy.1
+++ b/libraries/liblmdb/mdb_copy.1
@@ -13,6 +13,10 @@ mdb_copy \- LMDB environment copy tool
 .BR \-n ]
 [\c
 .BR \-v ]
+[\c
+.BI \-m \ module
+[\c
+.BI \-w \ password\fR]]
 .B srcpath
 [\c
 .BR dstpath ]
@@ -46,6 +50,15 @@ Open LDMB environment(s) which do not use subdirectories.
 .BR \-v
 Use the previous environment state instead of the latest state.
 This may be useful if the latest state has been corrupted.
+.TP
+.BI \-m \ module
+Load the specified dynamic module to utilize cryptographic functions.
+This is required to operate on environments that have been configured
+with page-level checksums or encryption.
+.TP
+.BI \-w \ password
+Specify the password for an encrypted environment. This is only
+used if a cryptography module has been loaded.
 
 .SH DIAGNOSTICS
 Exit status is zero if no errors occur.
diff --git a/libraries/liblmdb/mdb_drop.1 b/libraries/liblmdb/mdb_drop.1
index ff83fd98b5..877988bba7 100644
--- a/libraries/liblmdb/mdb_drop.1
+++ b/libraries/liblmdb/mdb_drop.1
@@ -12,6 +12,10 @@ mdb_drop \- LMDB database delete tool
 [\c
 .BR \-d ]
 [\c
+.BI \-m \ module
+[\c
+.BI \-w \ password\fR]]
+[\c
 .BI \-s \ subdb\fR]
 .BR \ envpath
 .SH DESCRIPTION
@@ -30,6 +34,15 @@ Operate on an LMDB database which does not use subdirectories.
 .BR \-d
 Delete the specified database, don't just empty it.
 .TP
+.BI \-m \ module
+Load the specified dynamic module to utilize cryptographic functions.
+This is required to operate on environments that have been configured
+with page-level checksums or encryption.
+.TP
+.BI \-w \ password
+Specify the password for an encrypted environment. This is only
+used if a cryptography module has been loaded.
+.TP
 .BR \-s \ subdb
 Operate on a specific subdatabase. If no database is specified, only the main database is dropped.
 .SH DIAGNOSTICS
diff --git a/libraries/liblmdb/mdb_dump.1 b/libraries/liblmdb/mdb_dump.1
index bde2773300..cb0fab12f7 100644
--- a/libraries/liblmdb/mdb_dump.1
+++ b/libraries/liblmdb/mdb_dump.1
@@ -18,6 +18,10 @@ mdb_dump \- LMDB environment export tool
 [\c
 .BR \-p ]
 [\c
+.BI \-m \ module
+[\c
+.BI \-w \ password\fR]]
+[\c
 .BR \-a \ |
 .BI \-s \ subdb\fR]
 .BR \ envpath
@@ -57,6 +61,15 @@ Note: different systems may have different notions about what characters
 are considered printing characters, and databases dumped in this manner may
 be less portable to external systems. 
 .TP
+.BI \-m \ module
+Load the specified dynamic module to utilize cryptographic functions.
+This is required to operate on environments that have been configured
+with page-level checksums or encryption.
+.TP
+.BI \-w \ password
+Specify the password for an encrypted environment. This is only
+used if a cryptography module has been loaded.
+.TP
 .BR \-a
 Dump all of the subdatabases in the environment.
 .TP
diff --git a/libraries/liblmdb/mdb_dump.c b/libraries/liblmdb/mdb_dump.c
index 1b3d4dee83..bcedd006ce 100644
--- a/libraries/liblmdb/mdb_dump.c
+++ b/libraries/liblmdb/mdb_dump.c
@@ -326,6 +326,8 @@ txn_abort:
 	mdb_txn_abort(txn);
 env_close:
 	mdb_env_close(env);
+	if (mlm)
+		mlm_unload(mlm);
 
 	return rc ? EXIT_FAILURE : EXIT_SUCCESS;
 }
diff --git a/libraries/liblmdb/mdb_load.1 b/libraries/liblmdb/mdb_load.1
index 7dd8aa1f42..dbcbd9b189 100644
--- a/libraries/liblmdb/mdb_load.1
+++ b/libraries/liblmdb/mdb_load.1
@@ -12,6 +12,10 @@ mdb_load \- LMDB environment import tool
 [\c
 .BR \-n ]
 [\c
+.BI \-m \ module
+[\c
+.BI \-w \ password\fR]]
+[\c
 .BI \-s \ subdb\fR]
 [\c
 .BR \-N ]
@@ -50,6 +54,15 @@ Read from the specified file instead of from the standard input.
 .BR \-n
 Load an LMDB database which does not use subdirectories.
 .TP
+.BI \-m \ module
+Load the specified dynamic module to utilize cryptographic functions.
+This is required to operate on environments that have been configured
+with page-level checksums or encryption.
+.TP
+.BI \-w \ password
+Specify the password for an encrypted environment. This is only
+used if a cryptography module has been loaded.
+.TP
 .BR \-s \ subdb
 Load a specific subdatabase. If no database is specified, data is loaded into the main database.
 .TP
diff --git a/libraries/liblmdb/mdb_stat.1 b/libraries/liblmdb/mdb_stat.1
index a618450eff..2adceb825c 100644
--- a/libraries/liblmdb/mdb_stat.1
+++ b/libraries/liblmdb/mdb_stat.1
@@ -16,6 +16,10 @@ mdb_stat \- LMDB environment status tool
 [\c
 .BR \-v ]
 [\c
+.BI \-m \ module
+[\c
+.BI \-w \ password\fR]]
+[\c
 .BR \-r [ r ]]
 [\c
 .BR \-a \ |
@@ -45,6 +49,15 @@ Display the status of an LMDB database which does not use subdirectories.
 Use the previous environment state instead of the latest state.
 This may be useful if the latest state has been corrupted.
 .TP
+.BI \-m \ module
+Load the specified dynamic module to utilize cryptographic functions.
+This is required to operate on environments that have been configured
+with page-level checksums or encryption.
+.TP
+.BI \-w \ password
+Specify the password for an encrypted environment. This is only
+used if a cryptography module has been loaded.
+.TP
 .BR \-r
 Display information about the environment reader table.
 Shows the process ID, thread ID, and transaction ID for each active
diff --git a/libraries/liblmdb/mdb_stat.c b/libraries/liblmdb/mdb_stat.c
index 1bb1bcb342..49afb14198 100644
--- a/libraries/liblmdb/mdb_stat.c
+++ b/libraries/liblmdb/mdb_stat.c
@@ -269,6 +269,8 @@ txn_abort:
 	mdb_txn_abort(txn);
 env_close:
 	mdb_env_close(env);
+	if (mlm)
+		mlm_unload(mlm);
 
 	return rc ? EXIT_FAILURE : EXIT_SUCCESS;
 }
diff --git a/libraries/liblmdb/mtest_enc2.c b/libraries/liblmdb/mtest_enc2.c
index b69c935d79..e0e594f835 100644
--- a/libraries/liblmdb/mtest_enc2.c
+++ b/libraries/liblmdb/mtest_enc2.c
@@ -34,13 +34,11 @@ int main(int argc,char * argv[])
 	MDB_stat mst;
 	MDB_cursor *cursor, *cur2;
 	MDB_cursor_op op;
-	MDB_val enckey;
 	int count;
 	int *values;
 	char sval[32] = "";
 	char password[] = "This is my passphrase for now...";
-	char *ekey;
-	void *lm;
+	void *mlm;
 	char *errmsg;
 
 	srand(time(NULL));
@@ -52,28 +50,14 @@ int main(int argc,char * argv[])
 			values[i] = rand()%1024;
 	    }
     
-		lm = lm_load("./crypto.lm", NULL, &cf, &errmsg);
-		if (!lm) {
+		E(mdb_env_create(&env));
+		mlm = mlm_setup(env, "./crypto.lm", password, &errmsg);
+		if (!mlm) {
 			fprintf(stderr,"Failed to load crypto module: %s\n", errmsg);
 			exit(1);
 		}
-
-		E(mdb_env_create(&env));
 		E(mdb_env_set_maxreaders(env, 1));
 		E(mdb_env_set_mapsize(env, 10485760));
-		if (cf->mcf_sumfunc) {
-			E(mdb_env_set_checksum(env, cf->mcf_sumfunc, cf->mcf_sumsize));
-		}
-		if (cf->mcf_encfunc) {
-			ekey = malloc(cf->mcf_keysize);
-			enckey.mv_data = ekey;
-			enckey.mv_size = cf->mcf_keysize;
-			if (cf->mcf_str2key)
-				cf->mcf_str2key(password, &enckey);
-			else
-				strncpy(ekey, password, cf->mcf_keysize);
-			E(mdb_env_set_encrypt(env, cf->mcf_encfunc, &enckey, cf->mcf_esumsize));
-		}
 		E(mdb_env_open(env, "./testdb", 0 /*|MDB_NOSYNC*/, 0664));
 
 		E(mdb_txn_begin(env, NULL, 0, &txn));
@@ -199,7 +183,7 @@ int main(int argc,char * argv[])
 
 		mdb_dbi_close(env, dbi);
 		mdb_env_close(env);
-		lm_unload(lm);
+		mlm_unload(mlm);
 
 	return 0;
 }

From fccd990c936fdeff558042169f3ed946fcef2f43 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 12 Oct 2020 22:55:51 +0100
Subject: [PATCH 400/504] ITS#9364 fix doxygen comment

---
 libraries/liblmdb/lmdb.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 128031ac34..95ce278b6d 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -1760,6 +1760,7 @@ typedef struct MDB_crypto_funcs {
 	 * @return A pointer to a #MDB_crypto_funcs structure.
 	 */
 typedef MDB_crypto_funcs *(MDB_crypto_hooks)(void);
+/**	@} */
 
 #ifdef __cplusplus
 }

From 311f071c1f1f2b701202b4fbc1752196aac999bc Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Fri, 23 Oct 2020 17:47:15 +0100
Subject: [PATCH 401/504] Support db_pagesize in mdb_load

Since setting was added in de08119a296a4e8569fa04d7bfcbc7b3c2b41f2f
---
 libraries/liblmdb/mdb_load.c | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c
index c9b2ad93f7..499cd0a866 100644
--- a/libraries/liblmdb/mdb_load.c
+++ b/libraries/liblmdb/mdb_load.c
@@ -40,6 +40,8 @@ static MDB_envinfo info;
 static MDB_val kbuf, dbuf;
 static MDB_val k0buf;
 
+static unsigned int pagesize;
+
 #define Yu	MDB_PRIy(u)
 
 #define STRLENOF(s)	(sizeof(s)-1)
@@ -128,6 +130,17 @@ static void readhdr(void)
 					prog, lineno, (char *)dbuf.mv_data+STRLENOF("maxreaders="));
 				exit(EXIT_FAILURE);
 			}
+		} else if (!strncmp(dbuf.mv_data, "db_pagesize=", STRLENOF("db_pagesize="))) {
+			int i;
+			ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
+			if (ptr) *ptr = '\0';
+			i = sscanf((char *)dbuf.mv_data+STRLENOF("db_pagesize="),
+				"%u", &pagesize);
+			if (i != 1) {
+				fprintf(stderr, "%s: line %"Yu": invalid pagesize %s\n",
+					prog, lineno, (char *)dbuf.mv_data+STRLENOF("db_pagesize="));
+				exit(EXIT_FAILURE);
+			}
 		} else {
 			int i;
 			for (i=0; dbflags[i].bit; i++) {
@@ -384,6 +397,9 @@ int main(int argc, char *argv[])
 	if (info.me_mapsize)
 		mdb_env_set_mapsize(env, info.me_mapsize);
 
+	if (pagesize)
+		mdb_env_set_pagesize(env, pagesize);
+
 	if (info.me_mapaddr)
 		envflags |= MDB_FIXEDMAP;
 
@@ -443,6 +459,10 @@ int main(int argc, char *argv[])
 				fprintf(stderr, "%s: line %"Yu": failed to read key value\n", prog, lineno);
 				goto txn_abort;
 			}
+			if (!key.mv_size) {
+				fprintf(stderr, "%s: line %"Yu": zero-length key(ignored)\n", prog, lineno);
+				continue;
+			}
 
 			if (append) {
 				appflag = MDB_APPEND;

From 4a61bbd8b211e0c6be69dfaf3baa76a34bf6af19 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 21 Oct 2020 21:24:53 +0100
Subject: [PATCH 402/504] ITS#9376 Fixes for repeated deletes with xcursor

On DUPSORT DBs, must initialize xcursor regardless of whether
caller requested its data. Also in cursor_prev must check whether
cursor index is still within range before using it.
---
 libraries/liblmdb/mdb.c | 224 +++++++++++++++++++++-------------------
 1 file changed, 115 insertions(+), 109 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index b82c1447c4..bb9d3d389d 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7620,16 +7620,12 @@ skip:
 
 	if (F_ISSET(leaf->mn_flags, F_DUPDATA)) {
 		mdb_xcursor_init1(mc, leaf);
-	}
-	if (data) {
+		rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL);
+		if (rc != MDB_SUCCESS)
+			return rc;
+	} else if (data) {
 		if ((rc = mdb_node_read(mc, leaf, data)) != MDB_SUCCESS)
 			return rc;
-
-		if (F_ISSET(leaf->mn_flags, F_DUPDATA)) {
-			rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL);
-			if (rc != MDB_SUCCESS)
-				return rc;
-		}
 	}
 
 	MDB_GET_KEY(leaf, key);
@@ -7653,7 +7649,8 @@ mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
 
 	mp = mc->mc_pg[mc->mc_top];
 
-	if (mc->mc_db->md_flags & MDB_DUPSORT) {
+	if ((mc->mc_db->md_flags & MDB_DUPSORT) &&
+		mc->mc_ki[mc->mc_top] < NUMKEYS(mp)) {
 		leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]);
 		if (F_ISSET(leaf->mn_flags, F_DUPDATA)) {
 			if (op == MDB_PREV || op == MDB_PREV_DUP) {
@@ -7695,27 +7692,25 @@ mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
 	DPRINTF(("==> cursor points to page %"Yu" with %u keys, key index %u",
 	    mdb_dbg_pgno(mp), NUMKEYS(mp), mc->mc_ki[mc->mc_top]));
 
+	if (!IS_LEAF(mp))
+		return MDB_CORRUPTED;
+
 	if (IS_LEAF2(mp)) {
 		key->mv_size = mc->mc_db->md_pad;
 		key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size);
 		return MDB_SUCCESS;
 	}
 
-	mdb_cassert(mc, IS_LEAF(mp));
 	leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]);
 
 	if (F_ISSET(leaf->mn_flags, F_DUPDATA)) {
 		mdb_xcursor_init1(mc, leaf);
-	}
-	if (data) {
+		rc = mdb_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL);
+		if (rc != MDB_SUCCESS)
+			return rc;
+	} else if (data) {
 		if ((rc = mdb_node_read(mc, leaf, data)) != MDB_SUCCESS)
 			return rc;
-
-		if (F_ISSET(leaf->mn_flags, F_DUPDATA)) {
-			rc = mdb_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL);
-			if (rc != MDB_SUCCESS)
-				return rc;
-		}
 	}
 
 	MDB_GET_KEY(leaf, key);
@@ -7873,24 +7868,22 @@ set1:
 
 	if (F_ISSET(leaf->mn_flags, F_DUPDATA)) {
 		mdb_xcursor_init1(mc, leaf);
-	}
-	if (data) {
-		if (F_ISSET(leaf->mn_flags, F_DUPDATA)) {
-			if (op == MDB_SET || op == MDB_SET_KEY || op == MDB_SET_RANGE) {
-				rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL);
+		if (op == MDB_SET || op == MDB_SET_KEY || op == MDB_SET_RANGE) {
+			rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL);
+		} else {
+			int ex2, *ex2p;
+			if (op == MDB_GET_BOTH) {
+				ex2p = &ex2;
+				ex2 = 0;
 			} else {
-				int ex2, *ex2p;
-				if (op == MDB_GET_BOTH) {
-					ex2p = &ex2;
-					ex2 = 0;
-				} else {
-					ex2p = NULL;
-				}
-				rc = mdb_cursor_set(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_SET_RANGE, ex2p);
-				if (rc != MDB_SUCCESS)
-					return rc;
+				ex2p = NULL;
 			}
-		} else if (op == MDB_GET_BOTH || op == MDB_GET_BOTH_RANGE) {
+			rc = mdb_cursor_set(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_SET_RANGE, ex2p);
+			if (rc != MDB_SUCCESS)
+				return rc;
+		}
+	} else if (data) {
+		if (op == MDB_GET_BOTH || op == MDB_GET_BOTH_RANGE) {
 			MDB_val olddata;
 			MDB_cmp_func *dcmp;
 			if ((rc = mdb_node_read(mc, leaf, &olddata)) != MDB_SUCCESS)
@@ -7948,22 +7941,23 @@ mdb_cursor_first(MDB_cursor *mc, MDB_val *key, MDB_val *data)
 	mc->mc_ki[mc->mc_top] = 0;
 
 	if (IS_LEAF2(mc->mc_pg[mc->mc_top])) {
-		key->mv_size = mc->mc_db->md_pad;
-		key->mv_data = LEAF2KEY(mc->mc_pg[mc->mc_top], 0, key->mv_size);
+		if ( key ) {
+			key->mv_size = mc->mc_db->md_pad;
+			key->mv_data = LEAF2KEY(mc->mc_pg[mc->mc_top], 0, key->mv_size);
+		}
 		return MDB_SUCCESS;
 	}
 
-	if (data) {
-		if (F_ISSET(leaf->mn_flags, F_DUPDATA)) {
-			mdb_xcursor_init1(mc, leaf);
-			rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL);
-			if (rc)
-				return rc;
-		} else {
-			if ((rc = mdb_node_read(mc, leaf, data)) != MDB_SUCCESS)
-				return rc;
-		}
+	if (F_ISSET(leaf->mn_flags, F_DUPDATA)) {
+		mdb_xcursor_init1(mc, leaf);
+		rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL);
+		if (rc)
+			return rc;
+	} else if (data) {
+		if ((rc = mdb_node_read(mc, leaf, data)) != MDB_SUCCESS)
+			return rc;
 	}
+
 	MDB_GET_KEY(leaf, key);
 	return MDB_SUCCESS;
 }
@@ -7992,21 +7986,21 @@ mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data)
 	leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
 
 	if (IS_LEAF2(mc->mc_pg[mc->mc_top])) {
-		key->mv_size = mc->mc_db->md_pad;
-		key->mv_data = LEAF2KEY(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top], key->mv_size);
+		if (key) {
+			key->mv_size = mc->mc_db->md_pad;
+			key->mv_data = LEAF2KEY(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top], key->mv_size);
+		}
 		return MDB_SUCCESS;
 	}
 
-	if (data) {
-		if (F_ISSET(leaf->mn_flags, F_DUPDATA)) {
-			mdb_xcursor_init1(mc, leaf);
-			rc = mdb_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL);
-			if (rc)
-				return rc;
-		} else {
-			if ((rc = mdb_node_read(mc, leaf, data)) != MDB_SUCCESS)
-				return rc;
-		}
+	if (F_ISSET(leaf->mn_flags, F_DUPDATA)) {
+		mdb_xcursor_init1(mc, leaf);
+		rc = mdb_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL);
+		if (rc)
+			return rc;
+	} else if (data) {
+		if ((rc = mdb_node_read(mc, leaf, data)) != MDB_SUCCESS)
+			return rc;
 	}
 
 	MDB_GET_KEY(leaf, key);
@@ -8782,6 +8776,8 @@ mdb_cursor_del(MDB_cursor *mc, unsigned int flags)
 		return rc;
 
 	mp = mc->mc_pg[mc->mc_top];
+	if (!IS_LEAF(mp))
+		return MDB_CORRUPTED;
 	if (IS_LEAF2(mp))
 		goto del_key;
 	leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]);
@@ -10180,60 +10176,70 @@ mdb_cursor_del0(MDB_cursor *mc)
 		}
 	}
 	rc = mdb_rebalance(mc);
+	if (rc)
+		goto fail;
 
-	if (rc == MDB_SUCCESS) {
-		/* DB is totally empty now, just bail out.
-		 * Other cursors adjustments were already done
-		 * by mdb_rebalance and aren't needed here.
-		 */
-		if (!mc->mc_snum)
-			return rc;
-
-		mp = mc->mc_pg[mc->mc_top];
-		nkeys = NUMKEYS(mp);
-
-		/* Adjust other cursors pointing to mp */
-		for (m2 = mc->mc_txn->mt_cursors[dbi]; !rc && m2; m2=m2->mc_next) {
-			m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2;
-			if (! (m2->mc_flags & m3->mc_flags & C_INITIALIZED))
-				continue;
-			if (m3->mc_snum < mc->mc_snum)
-				continue;
-			if (m3->mc_pg[mc->mc_top] == mp) {
-				/* if m3 points past last node in page, find next sibling */
-				if (m3->mc_ki[mc->mc_top] >= mc->mc_ki[mc->mc_top]) {
-					if (m3->mc_ki[mc->mc_top] >= nkeys) {
-						rc = mdb_cursor_sibling(m3, 1);
-						if (rc == MDB_NOTFOUND) {
-							m3->mc_flags |= C_EOF;
-							rc = MDB_SUCCESS;
-							continue;
-						}
-					}
-					if (mc->mc_db->md_flags & MDB_DUPSORT) {
-						MDB_node *node = NODEPTR(m3->mc_pg[m3->mc_top], m3->mc_ki[m3->mc_top]);
-						/* If this node has dupdata, it may need to be reinited
-						 * because its data has moved.
-						 * If the xcursor was not initd it must be reinited.
-						 * Else if node points to a subDB, nothing is needed.
-						 * Else (xcursor was initd, not a subDB) needs mc_pg[0] reset.
-						 */
-						if (node->mn_flags & F_DUPDATA) {
-							if (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) {
-								if (!(node->mn_flags & F_SUBDATA))
-									m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node);
-							} else {
-								mdb_xcursor_init1(m3, node);
-								m3->mc_xcursor->mx_cursor.mc_flags |= C_DEL;
-							}
-						}
-					}
-				}
-			}
-		}
-		mc->mc_flags |= C_DEL;
+	/* DB is totally empty now, just bail out.
+	 * Other cursors adjustments were already done
+	 * by mdb_rebalance and aren't needed here.
+	 */
+	if (!mc->mc_snum) {
+		mc->mc_flags |= C_EOF;
+		return rc;
 	}
 
+	ki = mc->mc_ki[mc->mc_top];
+	mp = mc->mc_pg[mc->mc_top];
+	nkeys = NUMKEYS(mp);
+
+	/* Adjust other cursors pointing to mp */
+	for (m2 = mc->mc_txn->mt_cursors[dbi]; !rc && m2; m2=m2->mc_next) {
+		m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2;
+		if (!(m2->mc_flags & m3->mc_flags & C_INITIALIZED))
+			continue;
+		if (m3->mc_snum < mc->mc_snum)
+			continue;
+		if (m3->mc_pg[mc->mc_top] == mp) {
+			/* if m3 points past last node in page, find next sibling */
+			if (m3->mc_ki[mc->mc_top] >= nkeys) {
+				rc = mdb_cursor_sibling(m3, 1);
+				if (rc == MDB_NOTFOUND) {
+					m3->mc_flags |= C_EOF;
+					rc = MDB_SUCCESS;
+					continue;
+				}
+				if (rc)
+					goto fail;
+			}
+			if (m3->mc_ki[mc->mc_top] >= ki ||
+				/* moved to right sibling */ m3->mc_pg[mc->mc_top] != mp) {
+				if (m3->mc_xcursor && !(m3->mc_flags & C_EOF)) {
+					MDB_node *node = NODEPTR(m3->mc_pg[m3->mc_top], m3->mc_ki[m3->mc_top]);
+					/* If this node has dupdata, it may need to be reinited
+					 * because its data has moved.
+					 * If the xcursor was not initd it must be reinited.
+					 * Else if node points to a subDB, nothing is needed.
+					 * Else (xcursor was initd, not a subDB) needs mc_pg[0] reset.
+					 */
+					if (node->mn_flags & F_DUPDATA) {
+						if (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) {
+							if (!(node->mn_flags & F_SUBDATA))
+								m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node);
+						} else {
+							mdb_xcursor_init1(m3, node);
+							rc = mdb_cursor_first(&m3->mc_xcursor->mx_cursor, NULL, NULL);
+							if (rc)
+								goto fail;
+						}
+					}
+					m3->mc_xcursor->mx_cursor.mc_flags |= C_DEL;
+				}
+				m3->mc_flags |= C_DEL;
+			}
+		}
+	}
+
+fail:
 	if (rc)
 		mc->mc_txn->mt_flags |= MDB_TXN_ERROR;
 	return rc;

From 576ccd10ff2ab98c98b179aaf45329c72b77cf96 Mon Sep 17 00:00:00 2001
From: Kris Zyp <kriszyp@gmail.com>
Date: Tue, 20 Oct 2020 08:04:56 -0600
Subject: [PATCH 403/504] ITS#9371 Fix unknown size of void compilation errors

---
 libraries/liblmdb/mdb.c | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index bb9d3d389d..dbd46c48cf 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -6564,7 +6564,7 @@ mdb_rpage_encsum(MDB_env *env, MDB_ID3 *id3, unsigned rem, int numpgs)
 		unsigned short muse = id3->muse;
 		rc = mdb_rpage_decrypt(env, id3, rem, numpgs);
 		if (!rc && env->me_sumfunc && muse != id3->muse) {
-			MDB_page *p = (MDB_page *)(id3->menc + rem * env->me_psize);
+			MDB_page *p = (MDB_page *)((char *)id3->menc + rem * env->me_psize);
 			rc = mdb_page_chk_checksum(env, p, numpgs * env->me_psize);
 		}
 	} else {
@@ -6578,7 +6578,7 @@ mdb_rpage_encsum(MDB_env *env, MDB_ID3 *id3, unsigned rem, int numpgs)
 				bit = 1;
 
 			id3->muse |= (bit << rem);
-			p = (MDB_page *)(id3->mptr + rem * env->me_psize);
+			p = (MDB_page *)((char *)id3->mptr + rem * env->me_psize);
 			rc = mdb_page_chk_checksum(env, p, numpgs * env->me_psize);
 		}
 	}
@@ -6940,7 +6940,7 @@ static int mdb_page_encrypt(MDB_env *env, MDB_page *dp, MDB_page *encp, size_t s
 		in.mv_size -= env->me_esumsize;
 		out.mv_size -= env->me_esumsize;
 		enckeys[2].mv_size = env->me_esumsize;
-		enckeys[2].mv_data = out.mv_data + out.mv_size;
+		enckeys[2].mv_data = (char *)out.mv_data + out.mv_size;
 	} else {
 		enckeys[2].mv_size = 0;
 		enckeys[2].mv_data = 0;
@@ -6972,11 +6972,11 @@ static int mdb_rpage_decrypt(MDB_env *env, MDB_ID3 *id3, int rem, int numpgs)
 		in.mv_data = (char *)id3->mptr + rem * env->me_psize + xsize;
 		enckeys[0] = env->me_enckey;
 		enckeys[1].mv_size = xsize;
-		enckeys[1].mv_data = in.mv_data - xsize;
+		enckeys[1].mv_data = (char *)in.mv_data - xsize;
 		if (env->me_esumsize) {
 			in.mv_size -= env->me_esumsize;
 			enckeys[2].mv_size = env->me_esumsize;
-			enckeys[2].mv_data = in.mv_data + in.mv_size;
+			enckeys[2].mv_data = (char *)in.mv_data + in.mv_size;
 		} else {
 			enckeys[2].mv_size = 0;
 			enckeys[2].mv_data = 0;
@@ -6988,7 +6988,7 @@ static int mdb_rpage_decrypt(MDB_env *env, MDB_ID3 *id3, int rem, int numpgs)
 		else {
 			MDB_page *penc, *pclr;
 			penc = (MDB_page *)enckeys[1].mv_data;
-			pclr = (MDB_page *)(out.mv_data - xsize);
+			pclr = (MDB_page *)((char *)out.mv_data - xsize);
 			pclr->mp_pgno = penc->mp_pgno;
 			pclr->mp_txnid = penc->mp_txnid;
 		}
@@ -7020,7 +7020,7 @@ static void mdb_page_set_checksum(MDB_env *env, MDB_page *mp, size_t size)
 	src.mv_size = size - env->me_sumsize;
 	src.mv_data = mp;
 	dst.mv_size = env->me_sumsize;
-	dst.mv_data = src.mv_data + src.mv_size;
+	dst.mv_data = (char *)src.mv_data + src.mv_size;
 	if (env->me_encfunc)
 		key = &env->me_enckey;
 	else
@@ -7035,7 +7035,7 @@ static int mdb_page_chk_checksum(MDB_env *env, MDB_page *mp, size_t size)
 	src.mv_size = size - env->me_sumsize;
 	src.mv_data = mp;
 	chk.mv_size = env->me_sumsize;
-	chk.mv_data = src.mv_data + src.mv_size;
+	chk.mv_data = (char *)src.mv_data + src.mv_size;
 	dst.mv_size = env->me_sumsize;
 	dst.mv_data = sumbuf;
 	if (env->me_encfunc)

From d85fe32dab55b88cec25fc73dadc6d38df807cce Mon Sep 17 00:00:00 2001
From: Kris Zyp <kriszyp@gmail.com>
Date: Tue, 20 Oct 2020 08:52:08 -0600
Subject: [PATCH 404/504] ITS#9372 Default to using POSIX semaphores on MacOS

Unless robust is specified (then use SysV)
---
 libraries/liblmdb/mdb.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index dbd46c48cf..b12b2b068f 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -164,6 +164,8 @@ typedef SSIZE_T	ssize_t;
 #if defined(__FreeBSD__) && defined(__FreeBSD_version) && __FreeBSD_version >= 1100110
 # define MDB_USE_POSIX_MUTEX	1
 # define MDB_USE_ROBUST	1
+#elif defined(__APPLE__) && !defined(MDB_USE_ROBUST)
+# define MDB_USE_POSIX_SEM	1
 #elif defined(__APPLE__) || defined (BSD) || defined(__FreeBSD_kernel__)
 # if !(defined(MDB_USE_POSIX_MUTEX) || defined(MDB_USE_POSIX_SEM))
 # define MDB_USE_SYSV_SEM	1

From 3a940d6860f715d950154a8c366616d48e9397ac Mon Sep 17 00:00:00 2001
From: Quanah Gibson-Mount <quanah@openldap.org>
Date: Tue, 12 Jan 2021 19:20:36 +0000
Subject: [PATCH 405/504] Happy New Year!

---
 libraries/liblmdb/COPYRIGHT      | 2 +-
 libraries/liblmdb/crypto.c       | 2 +-
 libraries/liblmdb/intro.doc      | 2 +-
 libraries/liblmdb/lmdb.h         | 2 +-
 libraries/liblmdb/mdb.c          | 2 +-
 libraries/liblmdb/mdb_copy.1     | 2 +-
 libraries/liblmdb/mdb_copy.c     | 2 +-
 libraries/liblmdb/mdb_drop.1     | 2 +-
 libraries/liblmdb/mdb_drop.c     | 2 +-
 libraries/liblmdb/mdb_dump.1     | 2 +-
 libraries/liblmdb/mdb_dump.c     | 2 +-
 libraries/liblmdb/mdb_load.1     | 2 +-
 libraries/liblmdb/mdb_load.c     | 2 +-
 libraries/liblmdb/mdb_stat.1     | 2 +-
 libraries/liblmdb/mdb_stat.c     | 2 +-
 libraries/liblmdb/midl.c         | 4 ++--
 libraries/liblmdb/midl.h         | 4 ++--
 libraries/liblmdb/module.c       | 2 +-
 libraries/liblmdb/module.h       | 2 +-
 libraries/liblmdb/mtest.c        | 2 +-
 libraries/liblmdb/mtest2.c       | 2 +-
 libraries/liblmdb/mtest3.c       | 2 +-
 libraries/liblmdb/mtest4.c       | 2 +-
 libraries/liblmdb/mtest5.c       | 2 +-
 libraries/liblmdb/mtest6.c       | 2 +-
 libraries/liblmdb/mtest_enc.c    | 2 +-
 libraries/liblmdb/mtest_enc2.c   | 2 +-
 libraries/liblmdb/mtest_remap.c  | 2 +-
 libraries/liblmdb/sample-bdb.txt | 2 +-
 libraries/liblmdb/sample-mdb.txt | 2 +-
 30 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/libraries/liblmdb/COPYRIGHT b/libraries/liblmdb/COPYRIGHT
index d9118b97c9..14eb1493d6 100644
--- a/libraries/liblmdb/COPYRIGHT
+++ b/libraries/liblmdb/COPYRIGHT
@@ -1,4 +1,4 @@
-Copyright 2011-2020 Howard Chu, Symas Corp.
+Copyright 2011-2021 Howard Chu, Symas Corp.
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/crypto.c b/libraries/liblmdb/crypto.c
index 40665cbacc..dadfa9244c 100644
--- a/libraries/liblmdb/crypto.c
+++ b/libraries/liblmdb/crypto.c
@@ -1,6 +1,6 @@
 /* crypto.c - LMDB encryption helper module */
 /*
- * Copyright 2020 Howard Chu, Symas Corp.
+ * Copyright 2020-2021 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/intro.doc b/libraries/liblmdb/intro.doc
index 4853af736f..b5bb06716a 100644
--- a/libraries/liblmdb/intro.doc
+++ b/libraries/liblmdb/intro.doc
@@ -1,5 +1,5 @@
 /*
- * Copyright 2015-2020 Howard Chu, Symas Corp.
+ * Copyright 2015-2021 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 95ce278b6d..be6bd8c4ef 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -136,7 +136,7 @@
  *
  *	@author	Howard Chu, Symas Corporation.
  *
- *	@copyright Copyright 2011-2020 Howard Chu, Symas Corp. All rights reserved.
+ *	@copyright Copyright 2011-2021 Howard Chu, Symas Corp. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted only as authorized by the OpenLDAP
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index b12b2b068f..265656a9dc 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5,7 +5,7 @@
  *	BerkeleyDB API, but much simplified.
  */
 /*
- * Copyright 2011-2020 Howard Chu, Symas Corp.
+ * Copyright 2011-2021 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mdb_copy.1 b/libraries/liblmdb/mdb_copy.1
index 762d3fbf57..db0c973a5e 100644
--- a/libraries/liblmdb/mdb_copy.1
+++ b/libraries/liblmdb/mdb_copy.1
@@ -1,5 +1,5 @@
 .TH MDB_COPY 1 "2017/07/31" "LMDB 0.9.90"
-.\" Copyright 2012-2020 Howard Chu, Symas Corp. All Rights Reserved.
+.\" Copyright 2012-2021 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_copy \- LMDB environment copy tool
diff --git a/libraries/liblmdb/mdb_copy.c b/libraries/liblmdb/mdb_copy.c
index c0a006c73f..c230caa2cb 100644
--- a/libraries/liblmdb/mdb_copy.c
+++ b/libraries/liblmdb/mdb_copy.c
@@ -1,6 +1,6 @@
 /* mdb_copy.c - memory-mapped database backup tool */
 /*
- * Copyright 2012-2020 Howard Chu, Symas Corp.
+ * Copyright 2012-2021 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mdb_drop.1 b/libraries/liblmdb/mdb_drop.1
index 877988bba7..36a73e1088 100644
--- a/libraries/liblmdb/mdb_drop.1
+++ b/libraries/liblmdb/mdb_drop.1
@@ -1,5 +1,5 @@
 .TH MDB_DROP 1 "2017/11/19" "LMDB 0.9.90"
-.\" Copyright 2014-2020 Howard Chu, Symas Corp. All Rights Reserved.
+.\" Copyright 2014-2021 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_drop \- LMDB database delete tool
diff --git a/libraries/liblmdb/mdb_drop.c b/libraries/liblmdb/mdb_drop.c
index 3b31beb1be..54e91332f0 100644
--- a/libraries/liblmdb/mdb_drop.c
+++ b/libraries/liblmdb/mdb_drop.c
@@ -1,6 +1,6 @@
 /* mdb_drop.c - memory-mapped database delete tool */
 /*
- * Copyright 2016-2020 Howard Chu, Symas Corp.
+ * Copyright 2016-2021 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mdb_dump.1 b/libraries/liblmdb/mdb_dump.1
index cb0fab12f7..acefe4e719 100644
--- a/libraries/liblmdb/mdb_dump.1
+++ b/libraries/liblmdb/mdb_dump.1
@@ -1,5 +1,5 @@
 .TH MDB_DUMP 1 "2017/07/31" "LMDB 0.9.90"
-.\" Copyright 2014-2020 Howard Chu, Symas Corp. All Rights Reserved.
+.\" Copyright 2014-2021 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_dump \- LMDB environment export tool
diff --git a/libraries/liblmdb/mdb_dump.c b/libraries/liblmdb/mdb_dump.c
index bcedd006ce..a3af117e02 100644
--- a/libraries/liblmdb/mdb_dump.c
+++ b/libraries/liblmdb/mdb_dump.c
@@ -1,6 +1,6 @@
 /* mdb_dump.c - memory-mapped database dump tool */
 /*
- * Copyright 2011-2020 Howard Chu, Symas Corp.
+ * Copyright 2011-2021 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mdb_load.1 b/libraries/liblmdb/mdb_load.1
index dbcbd9b189..aa25b03aad 100644
--- a/libraries/liblmdb/mdb_load.1
+++ b/libraries/liblmdb/mdb_load.1
@@ -1,5 +1,5 @@
 .TH MDB_LOAD 1 "2015/09/30" "LMDB 0.9.90"
-.\" Copyright 2014-2020 Howard Chu, Symas Corp. All Rights Reserved.
+.\" Copyright 2014-2021 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_load \- LMDB environment import tool
diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c
index 499cd0a866..62af83d883 100644
--- a/libraries/liblmdb/mdb_load.c
+++ b/libraries/liblmdb/mdb_load.c
@@ -1,6 +1,6 @@
 /* mdb_load.c - memory-mapped database load tool */
 /*
- * Copyright 2011-2020 Howard Chu, Symas Corp.
+ * Copyright 2011-2021 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mdb_stat.1 b/libraries/liblmdb/mdb_stat.1
index 2adceb825c..367bd6f36a 100644
--- a/libraries/liblmdb/mdb_stat.1
+++ b/libraries/liblmdb/mdb_stat.1
@@ -1,5 +1,5 @@
 .TH MDB_STAT 1 "2017/07/31" "LMDB 0.9.90"
-.\" Copyright 2012-2020 Howard Chu, Symas Corp. All Rights Reserved.
+.\" Copyright 2012-2021 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_stat \- LMDB environment status tool
diff --git a/libraries/liblmdb/mdb_stat.c b/libraries/liblmdb/mdb_stat.c
index 49afb14198..caf519b341 100644
--- a/libraries/liblmdb/mdb_stat.c
+++ b/libraries/liblmdb/mdb_stat.c
@@ -1,6 +1,6 @@
 /* mdb_stat.c - memory-mapped database status tool */
 /*
- * Copyright 2011-2020 Howard Chu, Symas Corp.
+ * Copyright 2011-2021 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/midl.c b/libraries/liblmdb/midl.c
index 049009e0da..272e557e3f 100644
--- a/libraries/liblmdb/midl.c
+++ b/libraries/liblmdb/midl.c
@@ -3,8 +3,8 @@
 /* $OpenLDAP$ */
 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
  *
- * Copyright 2000-2020 The OpenLDAP Foundation.
- * Portions Copyright 2001-2020 Howard Chu, Symas Corp.
+ * Copyright 2000-2021 The OpenLDAP Foundation.
+ * Portions Copyright 2001-2021 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/midl.h b/libraries/liblmdb/midl.h
index 48f4d82d23..aa45c2e726 100644
--- a/libraries/liblmdb/midl.h
+++ b/libraries/liblmdb/midl.h
@@ -11,8 +11,8 @@
 /* $OpenLDAP$ */
 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
  *
- * Copyright 2000-2020 The OpenLDAP Foundation.
- * Portions Copyright 2001-2020 Howard Chu, Symas Corp.
+ * Copyright 2000-2021 The OpenLDAP Foundation.
+ * Portions Copyright 2001-2021 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/module.c b/libraries/liblmdb/module.c
index 20e883beb8..52a24eb019 100644
--- a/libraries/liblmdb/module.c
+++ b/libraries/liblmdb/module.c
@@ -1,6 +1,6 @@
 /* module.c - helper for dynamically loading crypto module */
 /*
- * Copyright 2020 Howard Chu, Symas Corp.
+ * Copyright 2020-2021 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/module.h b/libraries/liblmdb/module.h
index 7c768bad11..13d4494dd8 100644
--- a/libraries/liblmdb/module.h
+++ b/libraries/liblmdb/module.h
@@ -1,6 +1,6 @@
 /* module.h - helper for dynamically loading crypto module */
 /*
- * Copyright 2020 Howard Chu, Symas Corp.
+ * Copyright 2020-2021 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest.c b/libraries/liblmdb/mtest.c
index 925f2d1ba7..2a45eb12c0 100644
--- a/libraries/liblmdb/mtest.c
+++ b/libraries/liblmdb/mtest.c
@@ -1,6 +1,6 @@
 /* mtest.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2020 Howard Chu, Symas Corp.
+ * Copyright 2011-2021 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest2.c b/libraries/liblmdb/mtest2.c
index db32525c5b..1ce4c9442d 100644
--- a/libraries/liblmdb/mtest2.c
+++ b/libraries/liblmdb/mtest2.c
@@ -1,6 +1,6 @@
 /* mtest2.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2020 Howard Chu, Symas Corp.
+ * Copyright 2011-2021 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest3.c b/libraries/liblmdb/mtest3.c
index bc471eeeaa..f8da0d331c 100644
--- a/libraries/liblmdb/mtest3.c
+++ b/libraries/liblmdb/mtest3.c
@@ -1,6 +1,6 @@
 /* mtest3.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2020 Howard Chu, Symas Corp.
+ * Copyright 2011-2021 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest4.c b/libraries/liblmdb/mtest4.c
index b7531755a9..3d7476c455 100644
--- a/libraries/liblmdb/mtest4.c
+++ b/libraries/liblmdb/mtest4.c
@@ -1,6 +1,6 @@
 /* mtest4.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2020 Howard Chu, Symas Corp.
+ * Copyright 2011-2021 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest5.c b/libraries/liblmdb/mtest5.c
index d6d1cf9cd7..d7a7307e29 100644
--- a/libraries/liblmdb/mtest5.c
+++ b/libraries/liblmdb/mtest5.c
@@ -1,6 +1,6 @@
 /* mtest5.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2020 Howard Chu, Symas Corp.
+ * Copyright 2011-2021 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest6.c b/libraries/liblmdb/mtest6.c
index e4d4e6b27e..cf8ba961d0 100644
--- a/libraries/liblmdb/mtest6.c
+++ b/libraries/liblmdb/mtest6.c
@@ -1,6 +1,6 @@
 /* mtest6.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2020 Howard Chu, Symas Corp.
+ * Copyright 2011-2021 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest_enc.c b/libraries/liblmdb/mtest_enc.c
index 266d5417b1..a8c11adfcb 100644
--- a/libraries/liblmdb/mtest_enc.c
+++ b/libraries/liblmdb/mtest_enc.c
@@ -1,6 +1,6 @@
 /* mtest_enc.c - memory-mapped database tester/toy with encryption */
 /*
- * Copyright 2011-2017 Howard Chu, Symas Corp.
+ * Copyright 2011-2021 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest_enc2.c b/libraries/liblmdb/mtest_enc2.c
index e0e594f835..853960ab52 100644
--- a/libraries/liblmdb/mtest_enc2.c
+++ b/libraries/liblmdb/mtest_enc2.c
@@ -1,6 +1,6 @@
 /* mtest_enc.c - memory-mapped database tester/toy with encryption */
 /*
- * Copyright 2011-2017 Howard Chu, Symas Corp.
+ * Copyright 2011-2021 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/mtest_remap.c b/libraries/liblmdb/mtest_remap.c
index 2e2b12c40f..93c062eb06 100644
--- a/libraries/liblmdb/mtest_remap.c
+++ b/libraries/liblmdb/mtest_remap.c
@@ -1,6 +1,6 @@
 /* mtest_remap.c - memory-mapped database tester/toy with page remapping */
 /*
- * Copyright 2011-2017 Howard Chu, Symas Corp.
+ * Copyright 2011-2021 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/sample-bdb.txt b/libraries/liblmdb/sample-bdb.txt
index c72078c722..8ca927c6cf 100644
--- a/libraries/liblmdb/sample-bdb.txt
+++ b/libraries/liblmdb/sample-bdb.txt
@@ -3,7 +3,7 @@
  * Do a line-by-line comparison of this and sample-mdb.txt
  */
 /*
- * Copyright 2012-2020 Howard Chu, Symas Corp.
+ * Copyright 2012-2021 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/sample-mdb.txt b/libraries/liblmdb/sample-mdb.txt
index e54a847068..2e1731631d 100644
--- a/libraries/liblmdb/sample-mdb.txt
+++ b/libraries/liblmdb/sample-mdb.txt
@@ -3,7 +3,7 @@
  * Do a line-by-line comparison of this and sample-bdb.txt
  */
 /*
- * Copyright 2012-2020 Howard Chu, Symas Corp.
+ * Copyright 2012-2021 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without

From 336d71714708ef02d6376621158ce0feb3578a5e Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 9 Feb 2021 23:38:06 +0000
Subject: [PATCH 406/504] ITS#9461 refix ITS#9376

Was setting C_DEL flag gratuitously
---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 265656a9dc..4cc425fcfa 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -10236,10 +10236,10 @@ mdb_cursor_del0(MDB_cursor *mc)
 					}
 					m3->mc_xcursor->mx_cursor.mc_flags |= C_DEL;
 				}
-				m3->mc_flags |= C_DEL;
 			}
 		}
 	}
+	m3->mc_flags |= C_DEL;
 
 fail:
 	if (rc)

From e9166d02d03cf719304e8df0ce6bb7d50a8c4db8 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 11 Feb 2021 11:34:57 +0000
Subject: [PATCH 407/504] ITS#9461 fix typo

---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 4cc425fcfa..e43c05b056 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -10239,7 +10239,7 @@ mdb_cursor_del0(MDB_cursor *mc)
 			}
 		}
 	}
-	m3->mc_flags |= C_DEL;
+	mc->mc_flags |= C_DEL;
 
 fail:
 	if (rc)

From 997300695ad61566cb02e8655ba4540e3c13a5a2 Mon Sep 17 00:00:00 2001
From: Quanah Gibson-Mount <quanah@openldap.org>
Date: Thu, 18 Feb 2021 16:25:45 +0000
Subject: [PATCH 408/504] ITS#9469 - Typo fixes

---
 libraries/liblmdb/lmdb.h | 4 ++--
 libraries/liblmdb/mdb.c  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index be6bd8c4ef..155bb2a3f6 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -652,7 +652,7 @@ int  mdb_env_create(MDB_env **env);
 	 *	<li>#MDB_NOTLS
 	 *		Don't use Thread-Local Storage. Tie reader locktable slots to
 	 *		#MDB_txn objects instead of to threads. I.e. #mdb_txn_reset() keeps
-	 *		the slot reseved for the #MDB_txn object. A thread may use parallel
+	 *		the slot reserved for the #MDB_txn object. A thread may use parallel
 	 *		read-only transactions. A read-only transaction may span threads if
 	 *		the user synchronizes its use. Applications that multiplex many
 	 *		user threads over individual OS threads need this option. Such an
@@ -1020,7 +1020,7 @@ void *mdb_env_get_userctx(MDB_env *env);
 typedef void MDB_assert_func(MDB_env *env, const char *msg);
 
 	/** Set or reset the assert() callback of the environment.
-	 * Disabled if liblmdb is buillt with NDEBUG.
+	 * Disabled if liblmdb is built with NDEBUG.
 	 * @note This hack should become obsolete as lmdb's error handling matures.
 	 * @param[in] env An environment handle returned by #mdb_env_create().
 	 * @param[in] func An #MDB_assert_func function, or 0.
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index e43c05b056..e284675b74 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -592,7 +592,7 @@ static txnid_t mdb_debug_start;
 	 *	The string is printed literally, with no format processing.
 	 */
 #define DPUTS(arg)	DPRINTF(("%s", arg))
-	/** Debuging output value of a cursor DBI: Negative in a sub-cursor. */
+	/** Debugging output value of a cursor DBI: Negative in a sub-cursor. */
 #define DDBI(mc) \
 	(((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi)
 /** @} */

From 6ad29167cc7468fa6b95822061e629e74987d28d Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sun, 14 Mar 2021 14:25:55 +0000
Subject: [PATCH 409/504] ITS#9376 simplify

---
 libraries/liblmdb/mdb.c | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index e284675b74..51fe5874db 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -10190,7 +10190,6 @@ mdb_cursor_del0(MDB_cursor *mc)
 		return rc;
 	}
 
-	ki = mc->mc_ki[mc->mc_top];
 	mp = mc->mc_pg[mc->mc_top];
 	nkeys = NUMKEYS(mp);
 
@@ -10202,19 +10201,18 @@ mdb_cursor_del0(MDB_cursor *mc)
 		if (m3->mc_snum < mc->mc_snum)
 			continue;
 		if (m3->mc_pg[mc->mc_top] == mp) {
+			if (m3->mc_ki[mc->mc_top] >= mc->mc_ki[mc->mc_top]) {
 			/* if m3 points past last node in page, find next sibling */
-			if (m3->mc_ki[mc->mc_top] >= nkeys) {
-				rc = mdb_cursor_sibling(m3, 1);
-				if (rc == MDB_NOTFOUND) {
-					m3->mc_flags |= C_EOF;
-					rc = MDB_SUCCESS;
-					continue;
+				if (m3->mc_ki[mc->mc_top] >= nkeys) {
+					rc = mdb_cursor_sibling(m3, 1);
+					if (rc == MDB_NOTFOUND) {
+						m3->mc_flags |= C_EOF;
+						rc = MDB_SUCCESS;
+						continue;
+					}
+					if (rc)
+						goto fail;
 				}
-				if (rc)
-					goto fail;
-			}
-			if (m3->mc_ki[mc->mc_top] >= ki ||
-				/* moved to right sibling */ m3->mc_pg[mc->mc_top] != mp) {
 				if (m3->mc_xcursor && !(m3->mc_flags & C_EOF)) {
 					MDB_node *node = NODEPTR(m3->mc_pg[m3->mc_top], m3->mc_ki[m3->mc_top]);
 					/* If this node has dupdata, it may need to be reinited

From fcf44d3fd8808011b325be7d0c57b2dfcb0499e1 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sun, 14 Mar 2021 14:29:44 +0000
Subject: [PATCH 410/504] ITS#9500 fix regression from ITS#8662

---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 51fe5874db..835ea3a718 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -8674,7 +8674,7 @@ put_sub:
 			xdata.mv_size = 0;
 			xdata.mv_data = "";
 			leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]);
-			if (flags == MDB_CURRENT) {
+			if ((flags & (MDB_CURRENT|MDB_APPENDDUP)) == MDB_CURRENT) {
 				xflags = MDB_CURRENT|MDB_NOSPILL;
 			} else {
 				mdb_xcursor_init1(mc, leaf);

From 557ab60606f3cbcc96f670318e4e88022869b33b Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Fri, 9 Apr 2021 14:06:33 +0100
Subject: [PATCH 411/504] ITS#9496 fix mdb_env_open bug from #8704

Broken in 3585a1eb977326c7e178c53f4eef1fdc81b46e63
---
 libraries/liblmdb/mdb.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 835ea3a718..a6cc3ab60f 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5361,9 +5361,6 @@ mdb_env_open2(MDB_env *env, int prev)
 #endif
 	env->me_maxpg = env->me_mapsize / env->me_psize;
 
-	if (env->me_txns)
-		env->me_txns->mti_txnid = meta.mm_txnid;
-
 #if MDB_DEBUG
 	{
 		MDB_meta *meta = mdb_env_pick_meta(env);
@@ -5463,6 +5460,9 @@ static int ESECT
 mdb_env_share_locks(MDB_env *env, int *excl)
 {
 	int rc = 0;
+	MDB_meta *meta = mdb_env_pick_meta(env);
+
+	env->me_txns->mti_txnid = meta->mm_txnid;
 
 #ifdef _WIN32
 	{

From 52836cdb8d7dabf9aac155dca25c316508d33ddb Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 7 Jun 2021 15:54:32 +0100
Subject: [PATCH 412/504] ITS#9574 add mdb_drop to .gitignore

---
 libraries/liblmdb/.gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libraries/liblmdb/.gitignore b/libraries/liblmdb/.gitignore
index d5102a87c0..80b6d81144 100644
--- a/libraries/liblmdb/.gitignore
+++ b/libraries/liblmdb/.gitignore
@@ -5,6 +5,7 @@ mdb_copy
 mdb_stat
 mdb_dump
 mdb_load
+mdb_drop
 *.lo
 *.[ao]
 *.so

From e2b82098fa592b20cb3ba79ddbf28f2b2a692e39 Mon Sep 17 00:00:00 2001
From: Kris Zyp <kriszyp@gmail.com>
Date: Wed, 28 Jul 2021 15:03:13 -0600
Subject: [PATCH 413/504] ITS#9618 fix Windows WRITEMAP flush

Revert back to using standard FlushViewOfFile/FlushFileBuffers
to sync data with WRITEMAP mode on Windows
---
 libraries/liblmdb/mdb.c | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index a6cc3ab60f..72c4c437ff 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -3008,9 +3008,9 @@ mdb_env_sync0(MDB_env *env, int force, pgno_t numpgs)
 	int rc = 0;
 	if (env->me_flags & MDB_RDONLY)
 		return EACCES;
-	if (force
-#ifndef _WIN32	/* Sync is normally achieved in Windows by doing WRITE_THROUGH writes */
-		|| !(env->me_flags & MDB_NOSYNC)
+	if (force || !(env->me_flags & MDB_NOSYNC)
+#ifdef _WIN32	/* Sync is normally achieved in Windows by doing WRITE_THROUGH writes */
+		&& (env->me_flags & MDB_WRITEMAP)
 #endif
 		) {
 		if (env->me_flags & MDB_WRITEMAP) {
@@ -3958,13 +3958,7 @@ mdb_page_flush(MDB_txn *txn, int keep)
 
 	j = i = keep;
 
-	if (env->me_flags & MDB_WRITEMAP
-#ifdef _WIN32
-		/* In windows, we still do writes to the file (with write-through enabled in sync mode),
-		 * as this is faster than FlushViewOfFile/FlushFileBuffers */
-		&& (env->me_flags & MDB_NOSYNC)
-#endif
-	) {
+	if (env->me_flags & MDB_WRITEMAP) {
 		goto done;
 	}
 

From 64fc67f4ae56fc0f67e069254a24747ae83814d6 Mon Sep 17 00:00:00 2001
From: Kris Zyp <kriszyp@gmail.com>
Date: Mon, 2 Nov 2020 20:33:09 +0000
Subject: [PATCH 414/504] ITS#9385 fix using MDB_NOSUBDIR with nonexistent file

---
 libraries/liblmdb/mdb.c | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 72c4c437ff..befc70bc73 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5995,11 +5995,8 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 #ifndef _WIN32
 	{
 		struct stat st;
-		rc = stat(path, &st);
-		if (rc)
-			return ErrCode();
 		flags &= ~MDB_RAWPART;
-		if (S_ISBLK(st.st_mode) || S_ISCHR(st.st_mode))
+		if (!stat(path, &st) && (S_ISBLK(st.st_mode) || S_ISCHR(st.st_mode)))
 			flags |= MDB_RAWPART | MDB_NOSUBDIR;
 	}
 #endif

From 0179cfab57d83ab1bec9e5bae4a3ac9101820e6e Mon Sep 17 00:00:00 2001
From: Joakim Hassila <jocke@ordo.one>
Date: Thu, 9 Jun 2022 21:48:38 +0200
Subject: [PATCH 415/504] ITS#9861 fix readonly regression from #9017

---
 libraries/liblmdb/mdb.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index befc70bc73..5bdec70b19 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -6074,9 +6074,11 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 		/* Synchronous fd for meta writes. Needed even with
 		 * MDB_NOSYNC/MDB_NOMETASYNC, in case these get reset.
 		 */
-		rc = mdb_fopen(env, &fname, MDB_O_META, mode, &env->me_mfd);
-		if (rc)
-			goto leave;
+		if (!(flags & (MDB_RDONLY|MDB_WRITEMAP))) {
+			rc = mdb_fopen(env, &fname, MDB_O_META, mode, &env->me_mfd);
+			if (rc)
+				goto leave;
+		}
 		DPRINTF(("opened dbenv %p", (void *) env));
 		if (excl > 0 && !(flags & MDB_PREVSNAPSHOT)) {
 			rc = mdb_env_share_locks(env, &excl);

From ad3f8367b0ef913dbff074ddd0fdcbcaa86fdbb0 Mon Sep 17 00:00:00 2001
From: NikoPLP <info@parlepeuple.fr>
Date: Thu, 1 Sep 2022 14:58:19 +0000
Subject: [PATCH 416/504] ITS#9910 fix undefined MDB_FDATASYNC on MacOSX

Broken by d85fe32 ITS#9372
---
 libraries/liblmdb/mdb.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 5bdec70b19..c9c39b0eb6 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -164,9 +164,10 @@ typedef SSIZE_T	ssize_t;
 #if defined(__FreeBSD__) && defined(__FreeBSD_version) && __FreeBSD_version >= 1100110
 # define MDB_USE_POSIX_MUTEX	1
 # define MDB_USE_ROBUST	1
-#elif defined(__APPLE__) && !defined(MDB_USE_ROBUST)
-# define MDB_USE_POSIX_SEM	1
 #elif defined(__APPLE__) || defined (BSD) || defined(__FreeBSD_kernel__)
+# if defined(__APPLE__) && !defined(MDB_USE_ROBUST)
+# define MSB_USE_POSIX_SEM	1
+# endif
 # if !(defined(MDB_USE_POSIX_MUTEX) || defined(MDB_USE_POSIX_SEM))
 # define MDB_USE_SYSV_SEM	1
 # endif

From 97e7e9ac795f77a1443cac2c534037dd7feb5dc5 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 21 Sep 2022 17:07:36 +0100
Subject: [PATCH 417/504] ITS#9524 fix loose page tracking

Fixes commit 0e17ba43a818f6bdab7759586e247bae12692c25, loose pages
that have been dropped from dirty list should no longer be counted.
---
 libraries/liblmdb/mdb.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index c9c39b0eb6..d4a2996be0 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -3717,7 +3717,6 @@ mdb_freelist_save(MDB_txn *txn)
 		unsigned x;
 		if ((rc = mdb_midl_need(&txn->mt_free_pgs, txn->mt_loose_count)) != 0)
 			return rc;
-		lost_loose = txn->mt_loose_count;
 		for (; mp; mp = NEXT_LOOSE_PAGE(mp)) {
 			mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno);
 			/* must also remove from dirty list */

From e51e6b82294ae97d8d6f76dc6ac051e0dcd6cfb7 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 21 Sep 2022 20:18:34 +0100
Subject: [PATCH 418/504] ITS#9910 fix typo

in ad3f8367b0ef913dbff074ddd0fdcbcaa86fdbb0
---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index d4a2996be0..3daa69b1e9 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -166,7 +166,7 @@ typedef SSIZE_T	ssize_t;
 # define MDB_USE_ROBUST	1
 #elif defined(__APPLE__) || defined (BSD) || defined(__FreeBSD_kernel__)
 # if defined(__APPLE__) && !defined(MDB_USE_ROBUST)
-# define MSB_USE_POSIX_SEM	1
+# define MDB_USE_POSIX_SEM	1
 # endif
 # if !(defined(MDB_USE_POSIX_MUTEX) || defined(MDB_USE_POSIX_SEM))
 # define MDB_USE_SYSV_SEM	1

From 17e13f7c07497ac0437c20602d7f1b0305a86435 Mon Sep 17 00:00:00 2001
From: Mike Hommey <mh@glandium.org>
Date: Thu, 6 Jun 2019 09:06:06 +0900
Subject: [PATCH 419/504] ITS#9030 - Use sys/cachectl.h rather than
 asm/cachectl.h on mips

It also contains the cacheflush function declaration.
---
 libraries/liblmdb/mdb.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 3daa69b1e9..c15d53690a 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -114,8 +114,7 @@ static NtCloseFunc *NtClose;
 
 #if defined(__mips) && defined(__linux)
 /* MIPS has cache coherency issues, requires explicit cache control */
-#include <asm/cachectl.h>
-extern int cacheflush(char *addr, int nbytes, int cache);
+#include <sys/cachectl.h>
 #define CACHEFLUSH(addr, bytes, cache)	cacheflush(addr, bytes, cache)
 #else
 #define CACHEFLUSH(addr, bytes, cache)

From 405ca7106bef7744d653432b256ea7386e6678ac Mon Sep 17 00:00:00 2001
From: Mike Hommey <mh@glandium.org>
Date: Fri, 4 Feb 2022 08:48:06 +0900
Subject: [PATCH 420/504] ITS#9919 - Mark infrequently used functions as cold
 rather than manually putting them in a separate section

---
 libraries/liblmdb/mdb.c | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index c15d53690a..e51d22db7d 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -251,15 +251,21 @@ union semun {
 # error "Two's complement, reasonably sized integer types, please"
 #endif
 
-#ifdef __GNUC__
-/** Put infrequently used env functions in separate section */
-# ifdef __APPLE__
-#  define	ESECT	__attribute__ ((section("__TEXT,text_env")))
-# else
-#  define	ESECT	__attribute__ ((section("text_env")))
-# endif
+#if (((__clang_major__ << 8) | __clang_minor__) >= 0x0302) || (((__GNUC__ << 8) | __GNUC_MINOR__) >= 0x0403)
+/** Mark infrequently used env functions as cold. This puts them in a separate
+ *  section, and optimizes them for size */
+#define ESECT __attribute__ ((cold))
 #else
-#define ESECT
+/* On older compilers, use a separate section */
+# ifdef __GNUC__
+#  ifdef __APPLE__
+#   define      ESECT   __attribute__ ((section("__TEXT,text_env")))
+#  else
+#   define      ESECT   __attribute__ ((section("text_env")))
+#  endif
+# else
+#  define ESECT
+# endif
 #endif
 
 #ifdef _WIN32

From 4031bdba882a34f0a1cffc210e9372537591cf09 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 28 Nov 2022 14:29:53 +0000
Subject: [PATCH 421/504] ITS#9806 LMDB page_split: key threshold depends on
 page size

32 was chosen for page size of 4KB. Not large nough for 16KB pages.
---
 libraries/liblmdb/mdb.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index e51d22db7d..5c684afe6c 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -10436,9 +10436,13 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 				mc->mc_ki[mc->mc_top] = x;
 			}
 		} else {
-			int psize, nsize, k;
+			int psize, nsize, k, keythresh;
+
 			/* Maximum free space in an empty page */
 			pmax = env->me_psize - PAGEHDRSZ;
+			/* Threshold number of keys considered "small" */
+			keythresh = env->me_psize >> 7;
+
 			if (IS_LEAF(mp))
 				nsize = mdb_leaf_size(env, newkey, newdata);
 			else
@@ -10479,7 +10483,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 			 * the split so the new page is emptier than the old page.
 			 * This yields better packing during sequential inserts.
 			 */
-			if (nkeys < 32 || nsize > pmax/16 || newindx >= nkeys) {
+			if (nkeys < keythresh || nsize > pmax/16 || newindx >= nkeys) {
 				/* Find split point */
 				psize = 0;
 				if (newindx <= split_indx || newindx >= nkeys) {

From 8e3cab0f633f11eff800170b2d1ada2b53616ab9 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 15 Nov 2022 15:33:48 +0000
Subject: [PATCH 422/504] ITS#9916 liblmdb: use alternate MDB_page2 struct for
 some accesses

fakepage pointers are only guaranteed to be 2-byte aligned. Use a
2-byte aligned struct definition when referencing 2-byte page members
if a page pointer possibly points to a fakepage.
---
 libraries/liblmdb/mdb.c | 129 +++++++++++++++++++++++-----------------
 1 file changed, 74 insertions(+), 55 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 5c684afe6c..4c960fe87c 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1046,9 +1046,26 @@ typedef struct MDB_page {
 #define mp_pad		mp_hdr.mh_pad
 #define mp_flags	mp_hdr.mh_flags
 #define mp_pb		mp_hdr.mh_pb
-	indx_t		mp_ptrs[1];		/**< dynamic size */
+	indx_t		mp_ptrs[0];		/**< dynamic size */
 } MDB_page;
 
+/** Alternate page header, for 2-byte aligned access */
+typedef struct MDB_page2 {
+	uint16_t	mp2_p[(sizeof(pgno_t)+sizeof(txnid_t))/2];
+	uint16_t	mp2_pad;
+	uint16_t	mp2_flags;
+	indx_t		mp2_lower;
+	indx_t		mp2_upper;
+	indx_t		mp2_ptrs[0];
+} MDB_page2;
+
+#define MP_PGNO(p)	(((MDB_page2 *)(void *)(p))->mp2_p)
+#define MP_PAD(p)	(((MDB_page2 *)(void *)(p))->mp2_pad)
+#define MP_FLAGS(p)	(((MDB_page2 *)(void *)(p))->mp2_flags)
+#define MP_LOWER(p)	(((MDB_page2 *)(void *)(p))->mp2_lower)
+#define MP_UPPER(p)	(((MDB_page2 *)(void *)(p))->mp2_upper)
+#define MP_PTRS(p)	(((MDB_page2 *)(void *)(p))->mp2_ptrs)
+
 	/** Size of the page header, excluding dynamic data at the end */
 #define PAGEHDRSZ	 ((unsigned)sizeof(MDB_page_header))
 
@@ -1059,10 +1076,10 @@ typedef struct MDB_page {
 #define	PAGEBASE	PAGEHDRSZ
 
 	/** Number of nodes on a page */
-#define NUMKEYS(p)	 (((p)->mp_lower - (PAGEHDRSZ-PAGEBASE)) >> 1)
+#define NUMKEYS(p)	 ((MP_LOWER(p) - (PAGEHDRSZ-PAGEBASE)) >> 1)
 
 	/** The amount of space remaining in the page */
-#define SIZELEFT(p)	 (indx_t)((p)->mp_upper - (p)->mp_lower)
+#define SIZELEFT(p)	 (indx_t)(MP_UPPER(p) - MP_LOWER(p))
 
 	/** The percentage of space used in the page, in tenths of a percent. */
 #define PAGEFILL(env, p) (1000L * ((env)->me_psize - PAGEHDRSZ - SIZELEFT(p)) / \
@@ -1073,15 +1090,15 @@ typedef struct MDB_page {
 #define FILL_THRESHOLD	 250
 
 	/** Test if a page is a leaf page */
-#define IS_LEAF(p)	 F_ISSET((p)->mp_flags, P_LEAF)
+#define IS_LEAF(p)	 F_ISSET(MP_FLAGS(p), P_LEAF)
 	/** Test if a page is a LEAF2 page */
-#define IS_LEAF2(p)	 F_ISSET((p)->mp_flags, P_LEAF2)
+#define IS_LEAF2(p)	 F_ISSET(MP_FLAGS(p), P_LEAF2)
 	/** Test if a page is a branch page */
-#define IS_BRANCH(p)	 F_ISSET((p)->mp_flags, P_BRANCH)
+#define IS_BRANCH(p)	 F_ISSET(MP_FLAGS(p), P_BRANCH)
 	/** Test if a page is an overflow page */
-#define IS_OVERFLOW(p)	 F_ISSET((p)->mp_flags, P_OVERFLOW)
+#define IS_OVERFLOW(p)	 F_ISSET(MP_FLAGS(p), P_OVERFLOW)
 	/** Test if a page is a sub page */
-#define IS_SUBP(p)	 F_ISSET((p)->mp_flags, P_SUBP)
+#define IS_SUBP(p)	 F_ISSET(MP_FLAGS(p), P_SUBP)
 
 	/** Test if (this non-sub page is dirty && env is non-#MDB_WRITEMAP) */
 #define IS_DIRTY_NW(txn, p)	((p)->mp_txnid > (txn)->mt_txnid)
@@ -1192,7 +1209,7 @@ typedef struct MDB_node {
 #define LEAFSIZE(k, d)	 (NODESIZE + (k)->mv_size + (d)->mv_size)
 
 	/** Address of node \b i in page \b p */
-#define NODEPTR(p, i)	 ((MDB_node *)((char *)(p) + (p)->mp_ptrs[i] + PAGEBASE))
+#define NODEPTR(p, i)	 ((MDB_node *)((char *)(p) + MP_PTRS(p)[i] + PAGEBASE))
 
 	/** Address of the key for the node */
 #define NODEKEY(node)	 (void *)((node)->mn_data)
@@ -1220,6 +1237,8 @@ typedef struct MDB_node {
 	/** Copy a page number from src to dst */
 #ifdef MISALIGNED_OK
 #define COPY_PGNO(dst,src)	dst = src
+#undef MP_PGNO
+#define MP_PGNO(p)	((p)->mp_pgno)
 #else
 #if MDB_SIZE_MAX > 0xffffffffU
 #define COPY_PGNO(dst,src)	do { \
@@ -1923,7 +1942,7 @@ static pgno_t
 mdb_dbg_pgno(MDB_page *mp)
 {
 	pgno_t ret;
-	COPY_PGNO(ret, mp->mp_pgno);
+	COPY_PGNO(ret, MP_PGNO(mp));
 	return ret;
 }
 
@@ -1976,7 +1995,7 @@ mdb_page_list(MDB_page *mp)
 	MDB_val key;
 	DKBUF;
 
-	switch (mp->mp_flags & (P_BRANCH|P_LEAF|P_LEAF2|P_META|P_OVERFLOW|P_SUBP)) {
+	switch (MP_FLAGS(mp) & (P_BRANCH|P_LEAF|P_LEAF2|P_META|P_OVERFLOW|P_SUBP)) {
 	case P_BRANCH:              type = "Branch page";		break;
 	case P_LEAF:                type = "Leaf page";			break;
 	case P_LEAF|P_SUBP:         type = "Sub-page";			break;
@@ -1990,7 +2009,7 @@ mdb_page_list(MDB_page *mp)
 			pgno, ((MDB_meta *)METADATA(mp))->mm_txnid);
 		return;
 	default:
-		fprintf(stderr, "Bad page %"Yu" flags 0x%X\n", pgno, mp->mp_flags);
+		fprintf(stderr, "Bad page %"Yu" flags 0x%X\n", pgno, MP_FLAGS(mp));
 		return;
 	}
 
@@ -2026,7 +2045,7 @@ mdb_page_list(MDB_page *mp)
 		total = EVEN(total);
 	}
 	fprintf(stderr, "Total: header %d + contents %d + unused %d\n",
-		IS_LEAF2(mp) ? PAGEHDRSZ : PAGEBASE + mp->mp_lower, total, SIZELEFT(mp));
+		IS_LEAF2(mp) ? PAGEHDRSZ : PAGEBASE + MP_LOWER(mp), total, SIZELEFT(mp));
 }
 
 void
@@ -7744,7 +7763,7 @@ mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data,
 			mc->mc_ki[mc->mc_top] = 0;
 			return MDB_NOTFOUND;
 		}
-		if (mp->mp_flags & P_LEAF2) {
+		if (MP_FLAGS(mp) & P_LEAF2) {
 			nodekey.mv_size = mc->mc_db->md_pad;
 			nodekey.mv_data = LEAF2KEY(mp, 0, nodekey.mv_size);
 		} else {
@@ -7765,7 +7784,7 @@ mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data,
 			unsigned int i;
 			unsigned int nkeys = NUMKEYS(mp);
 			if (nkeys > 1) {
-				if (mp->mp_flags & P_LEAF2) {
+				if (MP_FLAGS(mp) & P_LEAF2) {
 					nodekey.mv_data = LEAF2KEY(mp,
 						 nkeys-1, nodekey.mv_size);
 				} else {
@@ -7783,7 +7802,7 @@ mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data,
 				if (rc < 0) {
 					if (mc->mc_ki[mc->mc_top] < NUMKEYS(mp)) {
 						/* This is definitely the right page, skip search_page */
-						if (mp->mp_flags & P_LEAF2) {
+						if (MP_FLAGS(mp) & P_LEAF2) {
 							nodekey.mv_data = LEAF2KEY(mp,
 								 mc->mc_ki[mc->mc_top], nodekey.mv_size);
 						} else {
@@ -8348,7 +8367,7 @@ mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data,
 		*mc->mc_dbflag |= DB_DIRTY;
 		if ((mc->mc_db->md_flags & (MDB_DUPSORT|MDB_DUPFIXED))
 			== MDB_DUPFIXED)
-			np->mp_flags |= P_LEAF2;
+			MP_FLAGS(np) |= P_LEAF2;
 		mc->mc_flags |= C_INITIALIZED;
 	} else {
 		/* make sure all cursor pages are writable */
@@ -8370,7 +8389,7 @@ mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data,
 			fp_flags = P_LEAF;
 			fp = env->me_pbuf;
 			fp->mp_pad = data->mv_size; /* used if MDB_DUPFIXED */
-			fp->mp_lower = fp->mp_upper = (PAGEHDRSZ-PAGEBASE);
+			MP_LOWER(fp) = MP_UPPER(fp) = (PAGEHDRSZ-PAGEBASE);
 			olddata.mv_size = PAGEHDRSZ;
 			goto prep_subDB;
 		}
@@ -8445,18 +8464,18 @@ more:
 				dkey.mv_data = memcpy(fp+1, olddata.mv_data, olddata.mv_size);
 
 				/* Make sub-page header for the dup items, with dummy body */
-				fp->mp_flags = P_LEAF|P_SUBP;
-				fp->mp_lower = (PAGEHDRSZ-PAGEBASE);
+				MP_FLAGS(fp) = P_LEAF|P_SUBP;
+				MP_LOWER(fp) = (PAGEHDRSZ-PAGEBASE);
 				xdata.mv_size = PAGEHDRSZ + dkey.mv_size + data->mv_size;
 				if (mc->mc_db->md_flags & MDB_DUPFIXED) {
-					fp->mp_flags |= P_LEAF2;
+					MP_FLAGS(fp) |= P_LEAF2;
 					fp->mp_pad = data->mv_size;
 					xdata.mv_size += 2 * data->mv_size;	/* leave space for 2 more */
 				} else {
 					xdata.mv_size += 2 * (sizeof(indx_t) + NODESIZE) +
 						(dkey.mv_size & 1) + (data->mv_size & 1);
 				}
-				fp->mp_upper = xdata.mv_size - PAGEBASE;
+				MP_UPPER(fp) = xdata.mv_size - PAGEBASE;
 				olddata.mv_size = xdata.mv_size; /* pretend olddata is fp */
 			} else if (leaf->mn_flags & F_SUBDATA) {
 				/* Data is on sub-DB, just store it */
@@ -8479,7 +8498,7 @@ more:
 					}
 					/* FALLTHRU */ /* Big enough MDB_DUPFIXED sub-page */
 				case MDB_CURRENT:
-					COPY_PGNO(fp->mp_pgno, mp->mp_pgno);
+					COPY_PGNO(MP_PGNO(fp), MP_PGNO(mp));
 					mc->mc_xcursor->mx_cursor.mc_pg[0] = fp;
 					flags |= F_DUPDATA;
 					goto put_sub;
@@ -8487,7 +8506,7 @@ more:
 				xdata.mv_size = olddata.mv_size + offset;
 			}
 
-			fp_flags = fp->mp_flags;
+			fp_flags = MP_FLAGS(fp);
 			if (NODESIZE + NODEKSZ(leaf) + xdata.mv_size > env->me_nodemax) {
 					/* Too big for a sub-page, convert to sub-DB */
 					fp_flags &= ~P_SUBP;
@@ -8518,16 +8537,16 @@ prep_subDB:
 					sub_root = mp;
 			}
 			if (mp != fp) {
-				mp->mp_flags = fp_flags;
-				mp->mp_pad   = fp->mp_pad;
-				mp->mp_lower = fp->mp_lower;
-				mp->mp_upper = fp->mp_upper + offset;
+				MP_FLAGS(mp) = fp_flags;
+				MP_PAD(mp)   = MP_PAD(fp);
+				MP_LOWER(mp) = MP_LOWER(fp);
+				MP_UPPER(mp) = MP_UPPER(fp) + offset;
 				if (fp_flags & P_LEAF2) {
 					memcpy(METADATA(mp), METADATA(fp), NUMKEYS(fp) * fp->mp_pad);
 				} else {
-					memcpy((char *)mp + mp->mp_upper + PAGEBASE, (char *)fp + fp->mp_upper + PAGEBASE,
-						olddata.mv_size - fp->mp_upper - PAGEBASE);
-					memcpy((char *)(&mp->mp_ptrs), (char *)(&fp->mp_ptrs), NUMKEYS(fp) * sizeof(mp->mp_ptrs[0]));
+					memcpy((char *)mp + MP_UPPER(mp) + PAGEBASE, (char *)fp + MP_UPPER(fp) + PAGEBASE,
+						olddata.mv_size - MP_UPPER(fp) - PAGEBASE);
+					memcpy((char *)MP_PTRS(mp), (char *)MP_PTRS(fp), NUMKEYS(fp) * sizeof(mp->mp_ptrs[0]));
 					for (i=0; i<NUMKEYS(fp); i++)
 						mp->mp_ptrs[i] += offset;
 				}
@@ -8975,7 +8994,7 @@ mdb_node_add(MDB_cursor *mc, indx_t indx,
 	void		*ndata;
 	DKBUF;
 
-	mdb_cassert(mc, mp->mp_upper >= mp->mp_lower);
+	mdb_cassert(mc, MP_UPPER(mp) >= MP_LOWER(mp));
 
 	DPRINTF(("add to %s %spage %"Yu" index %i, data size %"Z"u key size %"Z"u [%s]",
 	    IS_LEAF(mp) ? "leaf" : "branch",
@@ -8994,8 +9013,8 @@ mdb_node_add(MDB_cursor *mc, indx_t indx,
 		memcpy(ptr, key->mv_data, ksize);
 
 		/* Just using these for counting */
-		mp->mp_lower += sizeof(indx_t);
-		mp->mp_upper -= ksize - sizeof(indx_t);
+		MP_LOWER(mp) += sizeof(indx_t);
+		MP_UPPER(mp) -= ksize - sizeof(indx_t);
 		return MDB_SUCCESS;
 	}
 
@@ -9032,14 +9051,14 @@ mdb_node_add(MDB_cursor *mc, indx_t indx,
 update:
 	/* Move higher pointers up one slot. */
 	for (i = NUMKEYS(mp); i > indx; i--)
-		mp->mp_ptrs[i] = mp->mp_ptrs[i - 1];
+		MP_PTRS(mp)[i] = MP_PTRS(mp)[i - 1];
 
 	/* Adjust free space offsets. */
-	ofs = mp->mp_upper - node_size;
-	mdb_cassert(mc, ofs >= mp->mp_lower + sizeof(indx_t));
-	mp->mp_ptrs[indx] = ofs;
-	mp->mp_upper = ofs;
-	mp->mp_lower += sizeof(indx_t);
+	ofs = MP_UPPER(mp) - node_size;
+	mdb_cassert(mc, ofs >= MP_LOWER(mp) + sizeof(indx_t));
+	MP_PTRS(mp)[indx] = ofs;
+	MP_UPPER(mp) = ofs;
+	MP_LOWER(mp) += sizeof(indx_t);
 
 	/* Write the node data. */
 	node = NODEPTR(mp, indx);
@@ -9081,7 +9100,7 @@ update:
 full:
 	DPRINTF(("not enough room in page %"Yu", got %u ptrs",
 		mdb_dbg_pgno(mp), NUMKEYS(mp)));
-	DPRINTF(("upper-lower = %u - %u = %"Z"d", mp->mp_upper,mp->mp_lower,room));
+	DPRINTF(("upper-lower = %u - %u = %"Z"d", MP_UPPER(mp),MP_LOWER(mp),room));
 	DPRINTF(("node size = %"Z"u", node_size));
 	mc->mc_txn->mt_flags |= MDB_TXN_ERROR;
 	return MDB_PAGE_FULL;
@@ -9112,8 +9131,8 @@ mdb_node_del(MDB_cursor *mc, int ksize)
 		base = LEAF2KEY(mp, indx, ksize);
 		if (x)
 			memmove(base, base + ksize, x * ksize);
-		mp->mp_lower -= sizeof(indx_t);
-		mp->mp_upper += ksize - sizeof(indx_t);
+		MP_LOWER(mp) -= sizeof(indx_t);
+		MP_UPPER(mp) += ksize - sizeof(indx_t);
 		return;
 	}
 
@@ -9127,21 +9146,21 @@ mdb_node_del(MDB_cursor *mc, int ksize)
 	}
 	sz = EVEN(sz);
 
-	ptr = mp->mp_ptrs[indx];
+	ptr = MP_PTRS(mp)[indx];
 	for (i = j = 0; i < numkeys; i++) {
 		if (i != indx) {
-			mp->mp_ptrs[j] = mp->mp_ptrs[i];
-			if (mp->mp_ptrs[i] < ptr)
-				mp->mp_ptrs[j] += sz;
+			MP_PTRS(mp)[j] = MP_PTRS(mp)[i];
+			if (MP_PTRS(mp)[i] < ptr)
+				MP_PTRS(mp)[j] += sz;
 			j++;
 		}
 	}
 
-	base = (char *)mp + mp->mp_upper + PAGEBASE;
-	memmove(base + sz, base, ptr - mp->mp_upper);
+	base = (char *)mp + MP_UPPER(mp) + PAGEBASE;
+	memmove(base + sz, base, ptr - MP_UPPER(mp));
 
-	mp->mp_lower -= sizeof(indx_t);
-	mp->mp_upper += sz;
+	MP_LOWER(mp) -= sizeof(indx_t);
+	MP_UPPER(mp) += sz;
 }
 
 /** Compact the main page after deleting a node on a subpage.
@@ -9170,11 +9189,11 @@ mdb_node_shrink(MDB_page *mp, indx_t indx)
 	} else {
 		xp = (MDB_page *)((char *)sp + delta); /* destination subpage */
 		for (i = NUMKEYS(sp); --i >= 0; )
-			xp->mp_ptrs[i] = sp->mp_ptrs[i] - delta;
+			MP_PTRS(xp)[i] = MP_PTRS(sp)[i] - delta;
 		len = PAGEHDRSZ;
 	}
-	sp->mp_upper = sp->mp_lower;
-	COPY_PGNO(sp->mp_pgno, mp->mp_pgno);
+	MP_UPPER(sp) = MP_LOWER(sp);
+	COPY_PGNO(MP_PGNO(sp), mp->mp_pgno);
 	SETDSZ(node, nsize);
 
 	/* Shift <lower nodes...initial part of subpage> upward */
@@ -9246,7 +9265,7 @@ mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node)
 		mx->mx_db.md_leaf_pages = 1;
 		mx->mx_db.md_overflow_pages = 0;
 		mx->mx_db.md_entries = NUMKEYS(fp);
-		COPY_PGNO(mx->mx_db.md_root, fp->mp_pgno);
+		COPY_PGNO(mx->mx_db.md_root, MP_PGNO(fp));
 		mx->mx_cursor.mc_snum = 1;
 		mx->mx_cursor.mc_top = 0;
 		mx->mx_cursor.mc_flags |= C_INITIALIZED;

From b9db2582cb31aa0ec88371db388095cc31ceb2f4 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 28 Nov 2022 23:56:43 +0000
Subject: [PATCH 423/504] ITS#9920 must account for size of authentication data

When computing amount of free space or fill factor in a page
---
 libraries/liblmdb/mdb.c | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 4c960fe87c..53732cc33b 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1082,8 +1082,8 @@ typedef struct MDB_page2 {
 #define SIZELEFT(p)	 (indx_t)(MP_UPPER(p) - MP_LOWER(p))
 
 	/** The percentage of space used in the page, in tenths of a percent. */
-#define PAGEFILL(env, p) (1000L * ((env)->me_psize - PAGEHDRSZ - SIZELEFT(p)) / \
-				((env)->me_psize - PAGEHDRSZ))
+#define PAGEFILL(env, p) (1000L * ((env)->me_pagespace - SIZELEFT(p)) / \
+				(env)->me_pagespace)
 	/** The minimum page fill factor, in tenths of a percent.
 	 *	Pages emptier than this are candidates for merging.
 	 */
@@ -1662,6 +1662,7 @@ struct MDB_env {
 	OVERLAPPED		*me_ov;			/**< Used for overlapping I/O requests */
 	int		me_ovs;				/**< Count of MDB_overlaps */
 #endif
+	int		me_pagespace;		/**< usable space in a page */
 #ifdef MDB_USE_POSIX_MUTEX	/* Posix mutexes reside in shared mem */
 #	define		me_rmutex	me_txns->mti_rmutex /**< Shared reader lock */
 #	define		me_wmutex	me_txns->mti_wmutex /**< Shared writer lock */
@@ -5305,6 +5306,11 @@ mdb_env_open2(MDB_env *env, int prev)
 	} else {
 		env->me_psize = meta.mm_psize;
 	}
+	env->me_pagespace = env->me_psize - PAGEHDRSZ
+#ifdef MDB_RPAGE_CACHE
+		- env->me_sumsize - env->me_esumsize
+#endif
+	;
 
 	/* Was a mapsize configured? */
 	if (!env->me_mapsize) {
@@ -5371,8 +5377,8 @@ mdb_env_open2(MDB_env *env, int prev)
 	}
 #endif
 
-	env->me_maxfree_1pg = (env->me_psize - PAGEHDRSZ) / sizeof(pgno_t) - 1;
-	env->me_nodemax = (((env->me_psize - PAGEHDRSZ) / MDB_MINKEYS) & -2)
+	env->me_maxfree_1pg = env->me_pagespace / sizeof(pgno_t) - 1;
+	env->me_nodemax = ((env->me_pagespace / MDB_MINKEYS) & -2)
 		- sizeof(indx_t);
 #if !(MDB_MAXKEYSIZE)
 	env->me_maxkey = env->me_nodemax - (NODESIZE + sizeof(MDB_db));
@@ -10458,7 +10464,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 			int psize, nsize, k, keythresh;
 
 			/* Maximum free space in an empty page */
-			pmax = env->me_psize - PAGEHDRSZ;
+			pmax = env->me_pagespace;
 			/* Threshold number of keys considered "small" */
 			keythresh = env->me_psize >> 7;
 

From 6264f539752159fa31834b419c441ec90e4fb57c Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 15 Dec 2022 09:36:58 +0000
Subject: [PATCH 424/504] ITS#9961 LMDB: fix MSVC error

---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 53732cc33b..ea1a2ea5c9 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -8612,7 +8612,7 @@ current:
 						 * Copy end of page, adjusting alignment so
 						 * compiler may copy words instead of bytes.
 						 */
-						off = (PAGEHDRSZ + data->mv_size) & -sizeof(size_t);
+						off = (PAGEHDRSZ + data->mv_size) & -(int)sizeof(size_t);
 						memcpy((size_t *)((char *)np + off),
 							(size_t *)((char *)omp + off), sz - off);
 						sz = PAGEHDRSZ;

From 406311ae59b3ab801837308c8a7b3efd59843bc3 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sun, 27 Aug 2023 15:50:51 +0100
Subject: [PATCH 425/504] ITS#10095 partial revert of ITS#9278
 2fd44e325195ae81664eb5dc36e7d265927c5ebc

The patch was incorrect and introduced numerous race conditions.
The original problem was a FreeBSD bug, subsequently fixed:
https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=269277
---
 libraries/liblmdb/mdb.c | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index ea1a2ea5c9..bda0093d71 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -6272,17 +6272,6 @@ mdb_env_close_active(MDB_env *env, int excl)
 			if (excl > 0)
 				semctl(env->me_rmutex->semid, 0, IPC_RMID);
 		}
-#elif defined(MDB_ROBUST_SUPPORTED)
-		/* If we have the filelock:  If we are the
-		 * only remaining user, clean up robust
-		 * mutexes.
-		 */
-		if (excl == 0)
-			mdb_env_excl_lock(env, &excl);
-		if (excl > 0) {
-			pthread_mutex_destroy(env->me_txns->mti_rmutex);
-			pthread_mutex_destroy(env->me_txns->mti_wmutex);
-		}
 #endif
 		munmap((void *)env->me_txns, (env->me_maxreaders-1)*sizeof(MDB_reader)+sizeof(MDB_txninfo));
 	}

From 741970078deacd3567e1e6d6b163a1a65758cf51 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sun, 5 Nov 2023 17:34:03 +0000
Subject: [PATCH 426/504] ITS#10125 mdb_load: fix cursor reinit in Append mode

---
 libraries/liblmdb/mdb_load.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c
index 62af83d883..19dd272c57 100644
--- a/libraries/liblmdb/mdb_load.c
+++ b/libraries/liblmdb/mdb_load.c
@@ -481,7 +481,7 @@ int main(int argc, char *argv[])
 			if (rc == MDB_KEYEXIST && putflags)
 				continue;
 			if (rc) {
-				fprintf(stderr, "mdb_cursor_put failed, error %d %s\n", rc, mdb_strerror(rc));
+				fprintf(stderr, "%s: line %"Yu": mdb_cursor_put failed, error %d %s\n", prog, lineno, rc, mdb_strerror(rc));
 				goto txn_abort;
 			}
 			batch++;
@@ -502,9 +502,11 @@ int main(int argc, char *argv[])
 					fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
 					goto txn_abort;
 				}
-				if (appflag & MDB_APPENDDUP) {
+				if (append) {
 					MDB_val k, d;
 					mdb_cursor_get(mc, &k, &d, MDB_LAST);
+					memcpy(prevk.mv_data, k.mv_data, k.mv_size);
+					prevk.mv_size = k.mv_size;
 				}
 				batch = 0;
 			}

From 22a41169c1f7a2c6a58c2fb10a4ed55bd8fe0d77 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= <clement@meilisearch.com>
Date: Mon, 27 Nov 2023 10:35:15 +0100
Subject: [PATCH 427/504] ITS#10137 LMDB: Allow users to define MDB_IDL_LOGN

---
 libraries/liblmdb/midl.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libraries/liblmdb/midl.h b/libraries/liblmdb/midl.h
index aa45c2e726..765708cf6b 100644
--- a/libraries/liblmdb/midl.h
+++ b/libraries/liblmdb/midl.h
@@ -56,7 +56,9 @@ typedef MDB_ID *MDB_IDL;
 /* IDL sizes - likely should be even bigger
  *   limiting factors: sizeof(ID), thread stack size
  */
+#ifndef MDB_IDL_LOGN
 #define	MDB_IDL_LOGN	16	/* DB_SIZE is 2^16, UM_SIZE is 2^17 */
+#endif
 #define MDB_IDL_DB_SIZE		(1<<MDB_IDL_LOGN)
 #define MDB_IDL_UM_SIZE		(1<<(MDB_IDL_LOGN+1))
 

From ab27be5f93e7921714fe72955ec6b377f6623920 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 25 Dec 2023 23:33:43 +0000
Subject: [PATCH 428/504] crypto demo: fixup OpenSSL 3 compat

---
 libraries/liblmdb/crypto.c | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/libraries/liblmdb/crypto.c b/libraries/liblmdb/crypto.c
index dadfa9244c..198e7a805b 100644
--- a/libraries/liblmdb/crypto.c
+++ b/libraries/liblmdb/crypto.c
@@ -46,11 +46,23 @@ typedef struct evp_cipher_ctx_st {
     /* FIXME: Should this even exist? It appears unused */
     void *app_data;             /* application stuff */
     int key_len;                /* May change for variable length cipher */
+#if OPENSSL_VERSION_NUMBER >= 0x30006000
+    int iv_len;                 /* IV length */
+#endif
     unsigned long flags;        /* Various flags */
     void *cipher_data;          /* per EVP data */
     int final_used;
     int block_mask;
     unsigned char final[EVP_MAX_BLOCK_LENGTH]; /* possible final block */
+
+#if OPENSSL_VERSION_NUMBER >= 0x30000000
+    /*
+     * Opaque ctx returned from a providers cipher algorithm implementation
+     * OSSL_FUNC_cipher_newctx()
+     */
+    void *algctx;
+    EVP_CIPHER *fetched_cipher;
+#endif
 } EVP_CIPHER_CTX;
 
 #define	CHACHA_KEY_SIZE	32

From b36e177c5ab41f5c22e4165358ac2df11bc6f7a7 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 26 Dec 2023 01:43:42 +0000
Subject: [PATCH 429/504] More crypto fixups

---
 libraries/liblmdb/Makefile |  2 +-
 libraries/liblmdb/crypto.c | 78 ++++++++++++++++++++++++++++++--------
 2 files changed, 64 insertions(+), 16 deletions(-)

diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile
index c252b50e21..fa9789223a 100644
--- a/libraries/liblmdb/Makefile
+++ b/libraries/liblmdb/Makefile
@@ -96,7 +96,7 @@ mtest_enc2:	  mtest_enc2.o module.o liblmdb.a crypto.lm
 	$(CC) $(LDFLAGS) -pthread -o $@ mtest_enc2.o module.o liblmdb.a $(LDL)
 
 crypto.lm:	crypto.c
-	$(CC) -shared -o $@ -lcrypto
+	$(CC) -shared $(CFLAGS) -o $@ $^ -lcrypto
 
 mdb.o: mdb.c lmdb.h midl.h
 	$(CC) $(CFLAGS) $(CPPFLAGS) -c mdb.c
diff --git a/libraries/liblmdb/crypto.c b/libraries/liblmdb/crypto.c
index 198e7a805b..fe98d3b979 100644
--- a/libraries/liblmdb/crypto.c
+++ b/libraries/liblmdb/crypto.c
@@ -32,8 +32,14 @@ static int mcf_str2key(const char *passwd, MDB_val *key)
 	return 0;
 }
 
-/* cheats - internal OpenSSL 1.1 structures */
-typedef struct evp_cipher_ctx_st {
+/* cheats - internal OpenSSL 1.1 structures
+ * These are copied from the OpenSSL source code.
+ *
+ * We use these to allow stack allocation of these structures
+ * and to prevent OpenSSL from malloc'ing and free'ing them,
+ * which would be too slow.
+ */
+typedef struct my_cipher_ctx_st {
     const EVP_CIPHER *cipher;
     ENGINE *engine;             /* functional reference if 'cipher' is
                                  * ENGINE-provided */
@@ -63,7 +69,17 @@ typedef struct evp_cipher_ctx_st {
     void *algctx;
     EVP_CIPHER *fetched_cipher;
 #endif
-} EVP_CIPHER_CTX;
+} MY_CIPHER_CTX;
+
+typedef struct evp_cipher_head {
+	int nid;
+	int block_size;
+	int key_len;
+	int iv_len;
+	unsigned long flags;
+	int (*init)(EVP_CIPHER_CTX *ctx, const unsigned char *key,
+		const unsigned char *iv, int enc);
+} evp_cipher_head;
 
 #define	CHACHA_KEY_SIZE	32
 #define CHACHA_CTR_SIZE	16
@@ -78,41 +94,67 @@ typedef struct {
     unsigned int  counter[CHACHA_CTR_SIZE / 4];
     unsigned char buf[CHACHA_BLK_SIZE];
     unsigned int  partial_len;
-} EVP_CHACHA_KEY;
+} MY_CHACHA_KEY;
 
 typedef struct {
-    EVP_CHACHA_KEY key;
+    MY_CHACHA_KEY key;
     unsigned int nonce[12/4];
     unsigned char tag[POLY1305_BLOCK_SIZE];
     unsigned char tls_aad[POLY1305_BLOCK_SIZE];
     struct { uint64_t aad, text; } len;
     int aad, mac_inited, tag_len, nonce_len;
     size_t tls_payload_length;
-} EVP_CHACHA_AEAD_CTX;
+} MY_CHACHA_AEAD_CTX;
+
+typedef struct {
+	double opaque[24];
+	unsigned int nonce[4];
+	unsigned char data[POLY1305_BLOCK_SIZE];
+	size_t num;
+	struct {
+		void (*foo1)();
+		void (*foo2)();
+	} func;
+} my_poly1305_ctx;
+
+typedef struct my_cipherdata {
+	MY_CHACHA_AEAD_CTX aead_ctx;
+	my_poly1305_ctx poly_ctx;
+} my_cipherdata;
 
 static int mcf_encfunc(const MDB_val *src, MDB_val *dst, const MDB_val *key, int encdec)
 {
 	unsigned char iv[12];
 	int ivl, outl, rc;
 	mdb_size_t *ptr;
-	EVP_CIPHER_CTX ctx = {0};
-	EVP_CHACHA_AEAD_CTX cactx;
+	MY_CIPHER_CTX myctx = {0};
+	EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)&myctx;
+	my_cipherdata cactx;
+	evp_cipher_head *eh = (evp_cipher_head *)cipher;
 
-	ctx.cipher_data = &cactx;
 	ptr = key[1].mv_data;
 	ivl = ptr[0] & 0xffffffff;
 	memcpy(iv, &ivl, 4);
 	memcpy(iv+4, ptr+1, sizeof(mdb_size_t));
-	EVP_CipherInit_ex(&ctx, cipher, NULL, key[0].mv_data, iv, encdec);
-	EVP_CIPHER_CTX_set_padding(&ctx, 0);
+	EVP_CipherInit_ex(ctx, cipher, NULL, NULL, NULL, encdec);
+
+	/* we can't set cipher_data before calling CipherInit because
+	 * that will just try to free it. So set it now, and then finish
+	 * up the other two Init calls that we disabled before.
+	 */
+	myctx.cipher_data = &cactx;
+	EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_INIT, 0, NULL);
+	eh->init(ctx, key[0].mv_data, iv, encdec);
+
+	EVP_CIPHER_CTX_set_padding(ctx, 0);
 	if (!encdec) {
-		EVP_CIPHER_CTX_ctrl(&ctx, EVP_CTRL_AEAD_SET_TAG, key[2].mv_size, key[2].mv_data);
+		EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_TAG, key[2].mv_size, key[2].mv_data);
 	}
-	rc = EVP_CipherUpdate(&ctx, dst->mv_data, &outl, src->mv_data, src->mv_size);
+	rc = EVP_CipherUpdate(ctx, dst->mv_data, &outl, src->mv_data, src->mv_size);
 	if (rc)
-		rc = EVP_CipherFinal_ex(&ctx, key[2].mv_data, &outl);
+		rc = EVP_CipherFinal_ex(ctx, key[2].mv_data, &outl);
 	if (rc && encdec) {
-		EVP_CIPHER_CTX_ctrl(&ctx, EVP_CTRL_AEAD_GET_TAG, key[2].mv_size, key[2].mv_data);
+		EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_GET_TAG, key[2].mv_size, key[2].mv_data);
 	}
 	return rc == 0;
 }
@@ -128,6 +170,12 @@ static const MDB_crypto_funcs mcf_table = {
 
 MDB_crypto_funcs *MDB_crypto()
 {
+	evp_cipher_head *eh;
 	cipher = (EVP_CIPHER *)EVP_chacha20_poly1305();
+
+	/* We must disable the implicit init calls */
+	eh = (evp_cipher_head *)cipher;
+	eh->flags &= ~(EVP_CIPH_CTRL_INIT|EVP_CIPH_ALWAYS_CALL_INIT);
+
 	return (MDB_crypto_funcs *)&mcf_table;
 }

From 9d45a80bf035729e035d1706f7608d6a95e00462 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Fri, 22 Dec 2023 13:14:32 +0000
Subject: [PATCH 430/504] ITS#9378 Add explicit replay logging

Logs essential ops so they can be replayed.
Ignores read ops for now.
---
 libraries/liblmdb/mdb.c | 138 ++++++++++++++++++++++++++++++++--------
 1 file changed, 112 insertions(+), 26 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index bda0093d71..358352b715 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -581,18 +581,26 @@ typedef MDB_ID	txnid_t;
 #define MDB_DEBUG 0
 #endif
 
+#define MDB_DBG_INFO	1
+#define MDB_DBG_TRACE	2
+
 #if MDB_DEBUG
-static int mdb_debug;
+static int mdb_debug = MDB_DBG_TRACE;
 static txnid_t mdb_debug_start;
 
 	/**	Print a debug message with printf formatting.
 	 *	Requires double parenthesis around 2 or more args.
 	 */
-# define DPRINTF(args) ((void) ((mdb_debug) && DPRINTF0 args))
+# define DPRINTF(args) ((void) ((mdb_debug & MDB_DBG_INFO) && DPRINTF0 args))
 # define DPRINTF0(fmt, ...) \
 	fprintf(stderr, "%s:%d " fmt "\n", mdb_func_, __LINE__, __VA_ARGS__)
+	/** Trace info for replaying */
+# define MDB_TRACE(args)	((void) ((mdb_debug & MDB_DBG_TRACE) && DPRINTF1 args))
+# define DPRINTF1(fmt, ...) \
+	fprintf(stderr, ">%d:%s: " fmt "\n", getpid(), mdb_func_, __VA_ARGS__)
 #else
 # define DPRINTF(args)	((void) 0)
+# define MDB_TRACE(args)	((void) 0)
 #endif
 	/**	Print a debug string.
 	 *	The string is printed literally, with no format processing.
@@ -693,6 +701,11 @@ static txnid_t mdb_debug_start;
 	 *	This is used for printing a hex dump of a key's contents.
 	 */
 #define DKBUF	char kbuf[DKBUF_MAXKEYSIZE*2+1]
+	/**	A data value buffer.
+	 *	@ingroup debug
+	 *	This is used for printing a hex dump of a #MDB_DUPSORT value's contents.
+	 */
+#define DDBUF	char dbuf[DKBUF_MAXKEYSIZE*2+1+2]
 	/**	Display a key in hex.
 	 *	@ingroup debug
 	 *	Invoke a function to display a key in hex.
@@ -700,6 +713,7 @@ static txnid_t mdb_debug_start;
 #define	DKEY(x)	mdb_dkey(x, kbuf)
 #else
 #define	DKBUF
+#define	DDBUF
 #define DKEY(x)	0
 #endif
 
@@ -1779,6 +1793,9 @@ static int	mdb_update_key(MDB_cursor *mc, MDB_val *key);
 static void	mdb_cursor_pop(MDB_cursor *mc);
 static int	mdb_cursor_push(MDB_cursor *mc, MDB_page *mp);
 
+static int	_mdb_cursor_del(MDB_cursor *mc, unsigned int flags);
+static int	_mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, unsigned int flags);
+
 static int	mdb_cursor_del0(MDB_cursor *mc);
 static int	mdb_del0(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data, unsigned flags);
 static int	mdb_cursor_sibling(MDB_cursor *mc, int move_right);
@@ -1977,6 +1994,18 @@ mdb_dkey(MDB_val *key, char *buf)
 	return buf;
 }
 
+static char *
+mdb_dval(MDB_txn *txn, MDB_dbi dbi, MDB_val *data, char *buf)
+{
+	if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) {
+		mdb_dkey(data, buf+1);
+		*buf = '[';
+		strcpy(buf + data->mv_size * 2 + 1, "]");
+	} else
+		*buf = '\0';
+	return buf;
+}
+
 static const char *
 mdb_leafnode_type(MDB_node *n)
 {
@@ -3293,7 +3322,7 @@ mdb_txn_renew0(MDB_txn *txn)
 		txn->mt_txnid++;
 #if MDB_DEBUG
 		if (txn->mt_txnid == mdb_debug_start)
-			mdb_debug = 1;
+			mdb_debug = MDB_DBG_INFO;
 #endif
 		txn->mt_child = NULL;
 		txn->mt_loose_pgs = NULL;
@@ -3520,6 +3549,7 @@ renew:
 			txn->mt_txnid, (flags & MDB_RDONLY) ? 'r' : 'w',
 			(void *) txn, (void *) env, txn->mt_dbs[MAIN_DBI].md_root));
 	}
+	MDB_TRACE(("%p, %p, %u = %p", env, parent, flags, txn));
 
 	return rc;
 }
@@ -3692,18 +3722,25 @@ mdb_txn_reset(MDB_txn *txn)
 	mdb_txn_end(txn, MDB_END_RESET);
 }
 
-void
-mdb_txn_abort(MDB_txn *txn)
+static void
+_mdb_txn_abort(MDB_txn *txn)
 {
 	if (txn == NULL)
 		return;
 
 	if (txn->mt_child)
-		mdb_txn_abort(txn->mt_child);
+		_mdb_txn_abort(txn->mt_child);
 
 	mdb_txn_end(txn, MDB_END_ABORT|MDB_END_SLOT|MDB_END_FREE);
 }
 
+void
+mdb_txn_abort(MDB_txn *txn)
+{
+	MDB_TRACE(("%p", txn));
+	_mdb_txn_abort(txn);
+}
+
 /** Save the freelist as of this transaction to the freeDB.
  * This changes the freelist. Keep trying until it stabilizes.
  *
@@ -3797,7 +3834,7 @@ mdb_freelist_save(MDB_txn *txn)
 			pglast = head_id = *(txnid_t *)key.mv_data;
 			total_room = head_room = 0;
 			mdb_tassert(txn, pglast <= env->me_pglast);
-			rc = mdb_cursor_del(&mc, 0);
+			rc = _mdb_cursor_del(&mc, 0);
 			if (rc)
 				return rc;
 		}
@@ -3817,7 +3854,7 @@ mdb_freelist_save(MDB_txn *txn)
 			do {
 				freecnt = free_pgs[0];
 				data.mv_size = MDB_IDL_SIZEOF(free_pgs);
-				rc = mdb_cursor_put(&mc, &key, &data, MDB_RESERVE);
+				rc = _mdb_cursor_put(&mc, &key, &data, MDB_RESERVE);
 				if (rc)
 					return rc;
 				/* Retry if mt_free_pgs[] grew during the Put() */
@@ -3866,7 +3903,7 @@ mdb_freelist_save(MDB_txn *txn)
 		key.mv_size = sizeof(head_id);
 		key.mv_data = &head_id;
 		data.mv_size = (head_room + 1) * sizeof(pgno_t);
-		rc = mdb_cursor_put(&mc, &key, &data, MDB_RESERVE);
+		rc = _mdb_cursor_put(&mc, &key, &data, MDB_RESERVE);
 		if (rc)
 			return rc;
 		/* IDL is initially empty, zero out at least the length */
@@ -3929,7 +3966,7 @@ mdb_freelist_save(MDB_txn *txn)
 			data.mv_data = mop -= len;
 			save = mop[0];
 			mop[0] = len;
-			rc = mdb_cursor_put(&mc, &key, &data, MDB_CURRENT);
+			rc = _mdb_cursor_put(&mc, &key, &data, MDB_CURRENT);
 			mop[0] = save;
 			if (rc || !mop_len)
 				break;
@@ -4206,8 +4243,8 @@ done:
 
 static int ESECT mdb_env_share_locks(MDB_env *env, int *excl);
 
-int
-mdb_txn_commit(MDB_txn *txn)
+static int
+_mdb_txn_commit(MDB_txn *txn)
 {
 	int		rc;
 	unsigned int i, end_mode;
@@ -4220,7 +4257,7 @@ mdb_txn_commit(MDB_txn *txn)
 	end_mode = MDB_END_EMPTY_COMMIT|MDB_END_UPDATE|MDB_END_SLOT|MDB_END_FREE;
 
 	if (txn->mt_child) {
-		rc = mdb_txn_commit(txn->mt_child);
+		rc = _mdb_txn_commit(txn->mt_child);
 		if (rc)
 			goto fail;
 	}
@@ -4401,7 +4438,7 @@ mdb_txn_commit(MDB_txn *txn)
 					goto fail;
 				}
 				data.mv_data = &txn->mt_dbs[i];
-				rc = mdb_cursor_put(&mc, &txn->mt_dbxs[i].md_name, &data,
+				rc = _mdb_cursor_put(&mc, &txn->mt_dbxs[i].md_name, &data,
 					F_SUBDATA);
 				if (rc)
 					goto fail;
@@ -4448,10 +4485,17 @@ done:
 	return MDB_SUCCESS;
 
 fail:
-	mdb_txn_abort(txn);
+	_mdb_txn_abort(txn);
 	return rc;
 }
 
+int
+mdb_txn_commit(MDB_txn *txn)
+{
+	MDB_TRACE(("%p", txn));
+	return _mdb_txn_commit(txn);
+}
+
 static int ESECT mdb_env_map(MDB_env *env, void *addr);
 
 /** Read the environment parameters of a DB environment before
@@ -4799,6 +4843,7 @@ mdb_env_create(MDB_env **env)
 	GET_PAGESIZE(e->me_os_psize);
 	VGMEMP_CREATE(e,0,0);
 	*env = e;
+	MDB_TRACE(("%p", e));
 	return MDB_SUCCESS;
 }
 
@@ -4966,6 +5011,7 @@ mdb_env_set_mapsize(MDB_env *env, mdb_size_t size)
 	env->me_mapsize = size;
 	if (env->me_psize)
 		env->me_maxpg = env->me_mapsize / env->me_psize;
+	MDB_TRACE(("%p, %"Yu"", env, size));
 	return MDB_SUCCESS;
 }
 
@@ -4975,6 +5021,7 @@ mdb_env_set_maxdbs(MDB_env *env, MDB_dbi dbs)
 	if (env->me_map)
 		return EINVAL;
 	env->me_maxdbs = dbs + CORE_DBS;
+	MDB_TRACE(("%p, %u", env, dbs));
 	return MDB_SUCCESS;
 }
 
@@ -4984,6 +5031,7 @@ mdb_env_set_maxreaders(MDB_env *env, unsigned int readers)
 	if (env->me_map || readers < 1)
 		return EINVAL;
 	env->me_maxreaders = readers;
+	MDB_TRACE(("%p, %u", env, readers));
 	return MDB_SUCCESS;
 }
 
@@ -6149,6 +6197,7 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 	}
 
 leave:
+	MDB_TRACE(("%p, %s, %u, %04o", env, path, flags & (CHANGEABLE|CHANGELESS), mode));
 	if (rc) {
 		mdb_env_close_active(env, excl);
 	}
@@ -6309,6 +6358,7 @@ mdb_env_close(MDB_env *env)
 	if (env == NULL)
 		return;
 
+	MDB_TRACE(("%p", env));
 	VGMEMP_DESTROY(env);
 	while ((dp = env->me_dpages) != NULL) {
 		VGMEMP_DEFINED(&dp->mp_next, sizeof(dp->mp_next));
@@ -8242,8 +8292,8 @@ mdb_cursor_touch(MDB_cursor *mc)
 /** Do not spill pages to disk if txn is getting full, may fail instead */
 #define MDB_NOSPILL	0x8000
 
-int
-mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data,
+static int
+_mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data,
     unsigned int flags)
 {
 	MDB_env		*env;
@@ -8698,7 +8748,7 @@ put_sub:
 			new_dupdata = (int)dkey.mv_size;
 			/* converted, write the original data first */
 			if (dkey.mv_size) {
-				rc = mdb_cursor_put(&mc->mc_xcursor->mx_cursor, &dkey, &xdata, xflags);
+				rc = _mdb_cursor_put(&mc->mc_xcursor->mx_cursor, &dkey, &xdata, xflags);
 				if (rc)
 					goto bad_sub;
 				/* we've done our job */
@@ -8726,7 +8776,7 @@ put_sub:
 			ecount = mc->mc_xcursor->mx_db.md_entries;
 			if (flags & MDB_APPENDDUP)
 				xflags |= MDB_APPEND;
-			rc = mdb_cursor_put(&mc->mc_xcursor->mx_cursor, data, &xdata, xflags);
+			rc = _mdb_cursor_put(&mc->mc_xcursor->mx_cursor, data, &xdata, xflags);
 			if (flags & F_SUBDATA) {
 				void *db = NODEDATA(leaf);
 				memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDB_db));
@@ -8767,7 +8817,20 @@ bad_sub:
 }
 
 int
-mdb_cursor_del(MDB_cursor *mc, unsigned int flags)
+mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data,
+    unsigned int flags)
+{
+	DKBUF;
+	DDBUF;
+	int rc = _mdb_cursor_put(mc, key, data, flags);
+	MDB_TRACE(("%p, %"Z"u[%s], %"Z"u%s, %u",
+		mc, key ? key->mv_size:0, DKEY(key), data ? data->mv_size:0,
+			data ? mdb_dval(mc->mc_txn, mc->mc_dbi, data, dbuf):"", flags));
+	return rc;
+}
+
+static int
+_mdb_cursor_del(MDB_cursor *mc, unsigned int flags)
 {
 	MDB_node	*leaf;
 	MDB_page	*mp;
@@ -8805,7 +8868,7 @@ mdb_cursor_del(MDB_cursor *mc, unsigned int flags)
 			if (!F_ISSET(leaf->mn_flags, F_SUBDATA)) {
 				mc->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf);
 			}
-			rc = mdb_cursor_del(&mc->mc_xcursor->mx_cursor, MDB_NOSPILL);
+			rc = _mdb_cursor_del(&mc->mc_xcursor->mx_cursor, MDB_NOSPILL);
 			if (rc)
 				return rc;
 			/* If sub-DB still has entries, we're done */
@@ -8869,6 +8932,14 @@ fail:
 	return rc;
 }
 
+int
+mdb_cursor_del(MDB_cursor *mc, unsigned int flags)
+{
+	MDB_TRACE(("%p, %u",
+		mc, flags));
+	return _mdb_cursor_del(mc, flags);
+}
+
 /** Allocate and initialize new pages for a database.
  * Set #MDB_TXN_ERROR on failure.
  * @param[in] mc a cursor on the database being added to.
@@ -9370,6 +9441,7 @@ mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **ret)
 		return ENOMEM;
 	}
 
+	MDB_TRACE(("%p, %u = %p", txn, dbi, mc));
 	*ret = mc;
 
 	return MDB_SUCCESS;
@@ -9433,6 +9505,7 @@ mdb_cursor_count(MDB_cursor *mc, mdb_size_t *countp)
 void
 mdb_cursor_close(MDB_cursor *mc)
 {
+	MDB_TRACE(("%p", mc));
 	if (mc) {
 		MDB_CURSOR_UNREF(mc, 0);
 	}
@@ -10261,6 +10334,8 @@ int
 mdb_del(MDB_txn *txn, MDB_dbi dbi,
     MDB_val *key, MDB_val *data)
 {
+	DKBUF;
+	DDBUF;
 	if (!key || !TXN_DBI_EXIST(txn, dbi, DB_USRVALID))
 		return EINVAL;
 
@@ -10272,6 +10347,9 @@ mdb_del(MDB_txn *txn, MDB_dbi dbi,
 		data = NULL;
 	}
 
+	MDB_TRACE(("%p, %u, %"Z"u[%s], %"Z"u%s",
+		txn, dbi, key ? key->mv_size:0, DKEY(key), data ? data->mv_size:0,
+		data ? mdb_dval(txn, dbi, data, dbuf):""));
 	return mdb_del0(txn, dbi, key, data, 0);
 }
 
@@ -10311,7 +10389,7 @@ mdb_del0(MDB_txn *txn, MDB_dbi dbi,
 		 */
 		mc.mc_next = txn->mt_cursors[dbi];
 		txn->mt_cursors[dbi] = &mc;
-		rc = mdb_cursor_del(&mc, flags);
+		rc = _mdb_cursor_del(&mc, flags);
 		txn->mt_cursors[dbi] = mc.mc_next;
 	}
 	return rc;
@@ -10756,6 +10834,8 @@ mdb_put(MDB_txn *txn, MDB_dbi dbi,
 	MDB_cursor mc;
 	MDB_xcursor mx;
 	int rc;
+	DKBUF;
+	DDBUF;
 
 	if (!key || !data || !TXN_DBI_EXIST(txn, dbi, DB_USRVALID))
 		return EINVAL;
@@ -10766,10 +10846,12 @@ mdb_put(MDB_txn *txn, MDB_dbi dbi,
 	if (txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_BLOCKED))
 		return (txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN;
 
+	MDB_TRACE(("%p, %u, %"Z"u[%s], %"Z"u%s, %u",
+		txn, dbi, key ? key->mv_size:0, DKEY(key), data->mv_size, mdb_dval(txn, dbi, data, dbuf), flags));
 	mdb_cursor_init(&mc, txn, dbi, &mx);
 	mc.mc_next = txn->mt_cursors[dbi];
 	txn->mt_cursors[dbi] = &mc;
-	rc = mdb_cursor_put(&mc, key, data, flags);
+	rc = _mdb_cursor_put(&mc, key, data, flags);
 	txn->mt_cursors[dbi] = mc.mc_next;
 	return rc;
 }
@@ -11175,7 +11257,7 @@ finish:
 		my.mc_error = rc;
 	mdb_env_cthr_toggle(&my, 1 | MDB_EOF);
 	rc = THREAD_FINISH(thr);
-	mdb_txn_abort(txn);
+	_mdb_txn_abort(txn);
 
 done:
 #ifdef _WIN32
@@ -11287,7 +11369,7 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd)
 	}
 
 leave:
-	mdb_txn_abort(txn);
+	_mdb_txn_abort(txn);
 	return rc;
 }
 
@@ -11552,6 +11634,7 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db
 			}
 		}
 		mdb_default_cmp(txn, MAIN_DBI);
+		MDB_TRACE(("%p, (null), %u = %u", txn, flags, MAIN_DBI));
 		return MDB_SUCCESS;
 	}
 
@@ -11613,7 +11696,7 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db
 		dummy.md_root = P_INVALID;
 		dummy.md_flags = flags & PERSISTENT_FLAGS;
 		WITH_CURSOR_TRACKING(mc,
-			rc = mdb_cursor_put(&mc, &key, &data, F_SUBDATA));
+			rc = _mdb_cursor_put(&mc, &key, &data, F_SUBDATA));
 		dbflag |= DB_DIRTY;
 	}
 
@@ -11638,6 +11721,7 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db
 		if (!unused) {
 			txn->mt_numdbs++;
 		}
+		MDB_TRACE(("%p, %s, %u = %u", txn, name, flags, slot));
 	}
 
 	return rc;
@@ -11669,6 +11753,7 @@ void mdb_dbi_close(MDB_env *env, MDB_dbi dbi)
 	ptr = env->me_dbxs[dbi].md_name.mv_data;
 	/* If there was no name, this was already closed */
 	if (ptr) {
+		MDB_TRACE(("%p, %u", env, dbi));
 		env->me_dbxs[dbi].md_name.mv_data = NULL;
 		env->me_dbxs[dbi].md_name.mv_size = 0;
 		env->me_dbflags[dbi] = 0;
@@ -11806,6 +11891,7 @@ int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del)
 	if (rc)
 		return rc;
 
+	MDB_TRACE(("%u, %d", dbi, del));
 	rc = mdb_drop0(mc, mc->mc_db->md_flags & MDB_DUPSORT);
 	/* Invalidate the dropped DB's cursors */
 	for (m2 = txn->mt_cursors[dbi]; m2; m2 = m2->mc_next)

From 009dd916863ccca9d1f56085a9ba19df69c7f306 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Fri, 22 Dec 2023 19:55:22 +0000
Subject: [PATCH 431/504] ITS#9378 Add replay tool

Reads a replay log and executes all the write ops
---
 libraries/liblmdb/Makefile |   2 +
 libraries/liblmdb/mplay.c  | 580 +++++++++++++++++++++++++++++++++++++
 2 files changed, 582 insertions(+)
 create mode 100644 libraries/liblmdb/mplay.c

diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile
index fa9789223a..7411b309f8 100644
--- a/libraries/liblmdb/Makefile
+++ b/libraries/liblmdb/Makefile
@@ -95,6 +95,8 @@ mtest_enc:    mtest_enc.o chacha8.o liblmdb.a
 mtest_enc2:	  mtest_enc2.o module.o liblmdb.a crypto.lm
 	$(CC) $(LDFLAGS) -pthread -o $@ mtest_enc2.o module.o liblmdb.a $(LDL)
 
+mplay:	mplay.o liblmdb.a
+
 crypto.lm:	crypto.c
 	$(CC) -shared $(CFLAGS) -o $@ $^ -lcrypto
 
diff --git a/libraries/liblmdb/mplay.c b/libraries/liblmdb/mplay.c
new file mode 100644
index 0000000000..5ca66d9176
--- /dev/null
+++ b/libraries/liblmdb/mplay.c
@@ -0,0 +1,580 @@
+/* mplay.c - memory-mapped database log replay */
+/*
+ * Copyright 2011-2023 Howard Chu, Symas Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <time.h>
+#include <string.h>
+#include <ctype.h>
+#include <assert.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+
+#include "lmdb.h"
+
+#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr)
+#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0))
+#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \
+	"%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort()))
+
+#define SCMP(s)	s, (sizeof(s)-1)
+char inbuf[8192];
+char *dbuf, *kbuf;
+size_t dbufsize;
+int maxkey;
+
+#define SOFF(s)	(sizeof(s)+1)
+
+#define MAXENVS	16
+#define MAXTXNS	16
+#define MAXCRSS	16
+
+#define MAXPIDS	16
+
+typedef struct crspair {
+	void *tcrs;	/* scanned text pointer */
+	MDB_cursor *rcrs;
+} crspair;
+
+typedef struct txnpair {
+	void *ttxn;	/* scanned text pointer */
+	MDB_txn *rtxn;
+	crspair cursors[MAXCRSS];
+	int ncursors;
+} txnpair;
+
+typedef struct envpair {
+	void *tenv;
+	MDB_env *renv;
+	txnpair txns[MAXTXNS];
+	int ntxns;
+} envpair;
+
+envpair envs[MAXENVS];
+int nenvs;
+
+envpair *lastenv;
+txnpair *lasttxn;
+crspair *lastcrs;
+
+typedef struct pidpair {
+	int tpid;
+	pid_t rpid;
+	int fdout, fdin;
+} pidpair;
+
+pidpair *lastpid;
+
+pidpair pids[MAXPIDS];
+int npids;
+
+unsigned long lcount;
+
+static int unhex(unsigned char *c2)
+{
+	int x, c;
+	x = *c2++ & 0x4f;
+	if (x & 0x40)
+		x -= 55;
+	c = x << 4;
+	x = *c2 & 0x4f;
+	if (x & 0x40)
+		x -= 55;
+	c |= x;
+	return c;
+}
+
+int inhex(char *in, char *out)
+{
+	char *c2 = out;
+	while (isxdigit(*in)) {
+		*c2++ = unhex((unsigned char *)in);
+		in += 2;
+	}
+	return c2 - out;
+}
+
+static void addenv(void *tenv, MDB_env *renv)
+{
+	assert(nenvs < MAXENVS);
+	envs[nenvs].tenv = tenv;
+	envs[nenvs].renv = renv;
+	envs[nenvs].ntxns = 0;
+	lastenv = envs+nenvs;
+	nenvs++;
+}
+
+static envpair *findenv(void *tenv)
+{
+	int i;
+	if (!lastenv || lastenv->tenv != tenv) {
+		for (i=0; i<nenvs; i++)
+			if (envs[i].tenv == tenv)
+				break;
+		assert(i < nenvs);
+		lastenv = &envs[i];
+	}
+	return lastenv;
+}
+
+static void delenv(envpair *ep)
+{
+	int i = ep - envs;
+	for (; i<nenvs-1; i++)
+		envs[i] = envs[i+1];
+	nenvs--;
+	lastenv = NULL;
+}
+
+static void addtxn(void *tenv, void *ttxn, MDB_txn *rtxn)
+{
+	envpair *ep;
+	txnpair *tp;
+
+	ep = findenv(tenv);
+	assert(ep->ntxns < MAXTXNS);
+	tp = ep->txns+ep->ntxns;
+	tp->ttxn = ttxn;
+	tp->rtxn = rtxn;
+	tp->ncursors = 0;
+	ep->ntxns++;
+	lasttxn = tp;
+}
+
+static txnpair *findtxn(void *ttxn)
+{
+	int i, j;
+	if (lasttxn && lasttxn->ttxn == ttxn)
+		return lasttxn;
+	if (lastenv) {
+		for (i=0; i<lastenv->ntxns; i++) {
+			if (lastenv->txns[i].ttxn == ttxn) {
+				lasttxn = lastenv->txns+i;
+				return lasttxn;
+			}
+		}
+	}
+	for (i=0; i<nenvs; i++) {
+		if (envs+i == lastenv) continue;
+		for (j=0; j<envs[i].ntxns; j++) {
+			if (envs[i].txns[j].ttxn == ttxn) {
+				lastenv = envs+i;
+				lasttxn = envs[i].txns+j;
+				return lasttxn;
+			}
+		}
+	}
+	assert(0);	/* should have found it */
+}
+
+static void deltxn(txnpair *tp)
+{
+	int i = tp - lastenv->txns;
+	for (; i<lastenv->ntxns-1; i++)
+		lastenv->txns[i] = lastenv->txns[i+1];
+	lastenv->ntxns--;
+	lasttxn = NULL;
+}
+
+static void addcrs(txnpair *tp, void *tcrs, MDB_cursor *rcrs)
+{
+	int j = tp->ncursors;
+	assert(tp->ncursors < MAXCRSS);
+
+	tp->cursors[j].tcrs = tcrs;
+	tp->cursors[j].rcrs = rcrs;
+	tp->ncursors++;
+	lastcrs = tp->cursors+j;
+}
+
+static crspair *findcrs(void *tcrs)
+{
+	int i, j, k;
+	envpair *ep;
+	txnpair *tp;
+	crspair *cp;
+	if (lastcrs && lastcrs->tcrs == tcrs)
+		return lastcrs;
+	if (lasttxn) {
+		for (k=0, cp=lasttxn->cursors; k<lasttxn->ncursors; k++, cp++) {
+			if (cp->tcrs == tcrs) {
+				lastcrs = cp;
+				return lastcrs;
+			}
+		}
+	}
+	if (lastenv) {
+		for (j=0, tp=lastenv->txns; j<lastenv->ntxns; j++, tp++){
+			if (tp == lasttxn)
+				continue;
+			for (k=0, cp = tp->cursors; k<tp->ncursors; k++, cp++) {
+				if (cp->tcrs == tcrs) {
+					lastcrs = cp;
+					lasttxn = tp;
+					return lastcrs;
+				}
+			}
+		}
+	}
+	for (i=0, ep=envs; i<nenvs; i++, ep++) {
+		for (j=0, tp=ep->txns; j<ep->ntxns; j++, tp++) {
+			if (tp == lasttxn)
+				continue;
+			for (k=0, cp = tp->cursors; k<tp->ncursors; k++, cp++) {
+				if (cp->tcrs == tcrs) {
+					lastcrs = cp;
+					lasttxn = tp;
+					lastenv = ep;
+					return lastcrs;
+				}
+			}
+		}
+	}
+	assert(0);	/* should have found it already */
+}
+
+static void delcrs(void *tcrs)
+{
+	int i;
+	crspair *cp = findcrs(tcrs);
+	mdb_cursor_close(cp->rcrs);
+	for (i = cp - lasttxn->cursors; i<lasttxn->ncursors-1; i++)
+		lasttxn->cursors[i] = lasttxn->cursors[i+1];
+	lasttxn->ncursors--;
+	lastcrs = NULL;
+}
+
+void child()
+{
+	int rc;
+	MDB_val key, data;
+	char *ptr;
+
+	while (fgets(inbuf, sizeof(inbuf), stdin)) {
+		ptr = inbuf;
+		if (!strncmp(ptr, SCMP("exit")))
+			break;
+
+		if (!strncmp(ptr, SCMP("mdb_env_create"))) {
+			void *tenv;
+			MDB_env *renv;
+			sscanf(ptr+SOFF("mdb_env_create"), "%p", &tenv);
+			E(mdb_env_create(&renv));
+			addenv(tenv, renv);
+		} else if (!strncmp(ptr, SCMP("mdb_env_set_maxdbs"))) {
+			void *tenv;
+			envpair *ep;
+			unsigned int maxdbs;
+			sscanf(ptr+SOFF("mdb_env_set_maxdbs"), "%p, %u", &tenv, &maxdbs);
+			ep = findenv(tenv);
+			E(mdb_env_set_maxdbs(ep->renv, maxdbs));
+		} else if (!strncmp(ptr, SCMP("mdb_env_set_mapsize"))) {
+			void *tenv;
+			envpair *ep;
+			mdb_size_t mapsize;
+			sscanf(ptr+SOFF("mdb_env_set_mapsize"), "%p, %"MDB_SCNy(u), &tenv, &mapsize);
+			ep = findenv(tenv);
+			E(mdb_env_set_mapsize(ep->renv, mapsize));
+		} else if (!strncmp(ptr, SCMP("mdb_env_open"))) {
+			void *tenv;
+			envpair *ep;
+			char *path;
+			int len;
+			unsigned int flags, mode;
+			sscanf(ptr+SOFF("mdb_env_open"), "%p, %n", &tenv, &len);
+			path = ptr+SOFF("mdb_env_open")+len;
+			ptr = strchr(path, ',');
+			*ptr++ = '\0';
+			sscanf(ptr, "%u, %o", &flags, &mode);
+			ep = findenv(tenv);
+			E(mdb_env_open(ep->renv, path, flags, mode));
+			if (!maxkey) {
+				maxkey = mdb_env_get_maxkeysize(ep->renv);
+				kbuf = malloc(maxkey+2);
+				dbuf = malloc(maxkey+2);
+				dbufsize = maxkey;
+			}
+		} else if (!strncmp(ptr, SCMP("mdb_env_close"))) {
+			void *tenv;
+			envpair *ep;
+			sscanf(ptr+SOFF("mdb_env_close"), "%p", &tenv);
+			ep = findenv(tenv);
+			mdb_env_close(ep->renv);
+			delenv(ep);
+			if (!nenvs)	/* if no other envs left, this process is done */
+				break;
+		} else if (!strncmp(ptr, SCMP("mdb_txn_begin"))) {
+			unsigned int flags;
+			void *tenv, *ttxn;
+			envpair *ep;
+			MDB_txn *rtxn;
+			sscanf(ptr+SOFF("mdb_txn_begin"), "%p, %*p, %u = %p", &tenv, &flags, &ttxn);
+			ep = findenv(tenv);
+			E(mdb_txn_begin(ep->renv, NULL, flags, &rtxn));
+			addtxn(tenv, ttxn, rtxn);
+		} else if (!strncmp(ptr, SCMP("mdb_txn_commit"))) {
+			void *ttxn;
+			txnpair *tp;
+			sscanf(ptr+SOFF("mdb_txn_commit"), "%p", &ttxn);
+			tp = findtxn(ttxn);
+			E(mdb_txn_commit(tp->rtxn));
+			deltxn(tp);
+		} else if (!strncmp(ptr, SCMP("mdb_txn_abort"))) {
+			void *ttxn;
+			txnpair *tp;
+			sscanf(ptr+SOFF("mdb_txn_abort"), "%p", &ttxn);
+			tp = findtxn(ttxn);
+			mdb_txn_abort(tp->rtxn);
+			deltxn(tp);
+		} else if (!strncmp(ptr, SCMP("mdb_dbi_open"))) {
+			void *ttxn;
+			txnpair *tp;
+			char *dbname;
+			unsigned int flags;
+			unsigned int tdbi;
+			MDB_dbi dbi;
+			sscanf(ptr+SOFF("mdb_dbi_open"), "%p, ", &ttxn);
+			dbname = strchr(ptr+SOFF("mdb_dbi_open"), ',');
+			dbname += 2;
+			ptr = strchr(dbname, ',');
+			*ptr++ = '\0';
+			if (!strcmp(dbname, "(null)"))
+				dbname = NULL;
+			sscanf(ptr, "%u = %u", &flags, &tdbi);
+			tp = findtxn(ttxn);
+			E(mdb_dbi_open(tp->rtxn, dbname, flags, &dbi));
+		} else if (!strncmp(ptr, SCMP("mdb_dbi_close"))) {
+			void *tenv;
+			envpair *ep;
+			unsigned int tdbi;
+			sscanf(ptr+SOFF("mdb_dbi_close"), "%p, %u", &tenv, &tdbi);
+			ep = findenv(tenv);
+			mdb_dbi_close(ep->renv, tdbi);
+		} else if (!strncmp(ptr, SCMP("mdb_cursor_open"))) {
+			void *ttxn, *tcrs;
+			txnpair *tp;
+			MDB_cursor *rcrs;
+			unsigned int tdbi;
+			sscanf(ptr+SOFF("mdb_cursor_open"), "%p, %u = %p", &ttxn, &tdbi, &tcrs);
+			tp = findtxn(ttxn);
+			E(mdb_cursor_open(tp->rtxn, tdbi, &rcrs));
+			addcrs(tp, tcrs, rcrs);
+		} else if (!strncmp(ptr, SCMP("mdb_cursor_close"))) {
+			void *tcrs;
+			sscanf(ptr+SOFF("mdb_cursor_close"), "%p", &tcrs);
+			delcrs(tcrs);
+		} else if (!strncmp(ptr, SCMP("mdb_cursor_put"))) {
+			void *tcrs;
+			crspair *cp;
+			unsigned int flags;
+			int len;
+			sscanf(ptr+SOFF("mdb_cursor_put"), "%p, ", &tcrs);
+			cp = findcrs(tcrs);
+			ptr = strchr(ptr+SOFF("mdb_cursor_put"), ',');
+			sscanf(ptr+1, "%"MDB_SCNy(u)",", &key.mv_size);
+			if (key.mv_size) {
+				ptr = strchr(ptr, '[');
+				inhex(ptr+1, kbuf);
+				key.mv_data = kbuf;
+				ptr += key.mv_size * 2 + 2;
+			}
+			ptr = strchr(ptr+1, ',');
+			sscanf(ptr+1, "%"MDB_SCNy(u)"%n", &data.mv_size, &len);
+			if (data.mv_size > dbufsize) {
+				dbuf = realloc(dbuf, data.mv_size+2);
+				assert(dbuf != NULL);
+				dbufsize = data.mv_size;
+			}
+			ptr += len+1;
+			if (*ptr == '[') {
+				inhex(ptr+1, dbuf);
+				data.mv_data = dbuf;
+				ptr += data.mv_size * 2 + 2;
+			} else {
+				sprintf(dbuf, "%09ld", (long)mdb_txn_id(lasttxn->rtxn));
+			}
+			sscanf(ptr+1, "%u", &flags);
+			E(mdb_cursor_put(cp->rcrs, &key, &data, flags));
+		} else if (!strncmp(ptr, SCMP("mdb_cursor_del"))) {
+			void *tcrs;
+			crspair *cp;
+			unsigned int flags;
+			sscanf(ptr+SOFF("mdb_cursor_del"), "%p, %u", &tcrs, &flags);
+			cp = findcrs(tcrs);
+			E(mdb_cursor_del(cp->rcrs, flags));
+		} else if (!strncmp(ptr, SCMP("mdb_put"))) {
+			void *ttxn;
+			txnpair *tp;
+			unsigned int tdbi, flags;
+			int len;
+			sscanf(ptr+SOFF("mdb_put"),"%p, %u, %"MDB_SCNy(u), &ttxn, &tdbi, &key.mv_size);
+			tp = findtxn(ttxn);
+			ptr = strchr(ptr+SOFF("mdb_put"), '[');
+			inhex(ptr+1, kbuf);
+			key.mv_data = kbuf;
+			ptr += key.mv_size * 2 + 2;
+			sscanf(ptr+1, "%"MDB_SCNy(u)"%n", &data.mv_size, &len);
+			if (data.mv_size > dbufsize) {
+				dbuf = realloc(dbuf, data.mv_size+2);
+				assert(dbuf != NULL);
+				dbufsize = data.mv_size;
+			}
+			ptr += len+1;
+			if (*ptr == '[') {
+				inhex(ptr+1, dbuf);
+				ptr += data.mv_size * 2 + 2;
+			} else {
+				sprintf(dbuf, "%09ld", (long)mdb_txn_id(tp->rtxn));
+			}
+			data.mv_data = dbuf;
+			sscanf(ptr+1, "%u", &flags);
+			RES(MDB_KEYEXIST,mdb_put(tp->rtxn, tdbi, &key, &data, flags));
+		} else if (!strncmp(ptr, SCMP("mdb_del"))) {
+			void *ttxn;
+			txnpair *tp;
+			unsigned int tdbi;
+			int len;
+			sscanf(ptr+SOFF("mdb_del"),"%p, %u, %"MDB_SCNy(u), &ttxn, &tdbi, &key.mv_size);
+			tp = findtxn(ttxn);
+			ptr = strchr(ptr+SOFF("mdb_del"), '[');
+			inhex(ptr+1, kbuf);
+			key.mv_data = kbuf;
+			ptr += key.mv_size * 2 + 2;
+			sscanf(ptr+1, "%"MDB_SCNy(u)"%n", &data.mv_size, &len);
+			if (data.mv_size > dbufsize) {
+				dbuf = realloc(dbuf, data.mv_size+2);
+				assert(dbuf != NULL);
+				dbufsize = data.mv_size;
+			}
+			ptr += len+1;
+			if (*ptr == '[') {
+				inhex(ptr+1, dbuf);
+			} else {
+				sprintf(dbuf, "%09ld", (long)mdb_txn_id(tp->rtxn));
+			}
+			data.mv_data = dbuf;
+			RES(MDB_NOTFOUND,mdb_del(tp->rtxn, tdbi, &key, &data));
+		}
+		write(1, "\n", 1);
+	}
+	exit(0);
+}
+
+static pidpair *addpid(int tpid)
+{
+	int fdout[2], fdin[2];
+	pid_t pid;
+	assert(npids < MAXPIDS);
+	pids[npids].tpid = tpid;
+	pipe(fdout);
+	pipe(fdin);
+	if ((pid = fork()) == 0) {
+		/* child */
+		fclose(stdin);
+		fclose(stdout);
+		dup2(fdout[0], 0);
+		dup2(fdin[1], 1);
+		stdin = fdopen(0, "r");
+		stdout = fdopen(1, "w");
+		child();
+		return 0;	/* NOTREACHED */
+	} else {
+		pids[npids].rpid = pid;
+		pids[npids].fdout = fdout[1];
+		pids[npids].fdin = fdin[0];
+		lastpid = pids+npids;
+		npids++;
+		return lastpid;
+	}
+}
+
+static pidpair *findpid(int tpid)
+{
+	int i;
+	if (!lastpid || lastpid->tpid != tpid) {
+		for (i=0; i<npids; i++)
+			if (pids[i].tpid == tpid)
+				break;
+		if (i == npids)
+			return NULL;
+		lastpid = &pids[i];
+	}
+	return lastpid;
+}
+
+volatile pid_t killpid;
+
+static void delpid(int tpid)
+{
+	pidpair *pp = findpid(tpid);
+	if (pp) {
+		pid_t kpid = pp->rpid;
+		killpid = kpid;
+		write(pp->fdout, "exit\n", sizeof("exit"));
+		while (killpid == kpid)
+			usleep(10000);
+	}
+}
+
+static void reaper(int sig)
+{
+	int status, i;
+	pid_t pid = waitpid(-1, &status, 0);
+	if (pid > 0) {
+		fprintf(stderr, "# %s %d\n", WIFEXITED(status) ? "exited" : "killed", pid);
+		for (i=0; i<npids; i++)
+			if (pids[i].rpid == pid)
+				break;
+		assert(i < npids);
+		close(pids[i].fdout);
+		close(pids[i].fdin);
+		for (;i<npids-1; i++)
+			pids[i] = pids[i+1];
+		npids--;
+		killpid = 0;
+	}
+}
+
+int main(int argc,char * argv[])
+{
+	signal(SIGCHLD, reaper);
+
+	while (fgets(inbuf, sizeof(inbuf), stdin)) {
+		pidpair *pp;
+		int tpid, len;
+		char c, *ptr;
+		lcount++;
+
+		if (inbuf[0] == '#' && !strncmp(inbuf+1, SCMP(" killed"))) {
+			sscanf(inbuf+SOFF("killed"),"%d", &tpid);
+			delpid(tpid);
+			continue;
+		}
+
+		if (inbuf[0] != '>')
+			continue;
+		ptr = inbuf+1;
+		sscanf(ptr, "%d:%n", &tpid, &len);
+		pp = findpid(tpid);
+		if (!pp)
+			pp = addpid(tpid);	/* new process */
+
+		ptr = inbuf+len+1;
+		len = strlen(ptr);
+		write(pp->fdout, ptr, len);	/* send command and wait for ack */
+		read(pp->fdin, &c, 1);
+	}
+	while (npids)
+		delpid(pids[0].tpid);
+}

From 0f8ee264c84a4ffc3c7c54fc9df9ab258d685493 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 26 Mar 2024 14:50:17 +0000
Subject: [PATCH 432/504] ITS#9037 mdb_page_search: fix error code when DBI
 record is missing

Use the more relevant MDB_BAD_DBI instead of MDB_NOTFOUND error code
---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 358352b715..d1149fc556 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -7334,7 +7334,7 @@ mdb_page_search(MDB_cursor *mc, MDB_val *key, int flags)
 					MDB_node *leaf = mdb_node_search(&mc2,
 						&mc->mc_dbx->md_name, &exact);
 					if (!exact)
-						return MDB_NOTFOUND;
+						return MDB_BAD_DBI;
 					if ((leaf->mn_flags & (F_DUPDATA|F_SUBDATA)) != F_SUBDATA)
 						return MDB_INCOMPATIBLE; /* not a named DB */
 					rc = mdb_node_read(&mc2, leaf, &data);

From dd2ad2fd5a81c8878a9f4575e6a5d119e37dcfb1 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 4 Apr 2024 07:15:24 +0100
Subject: [PATCH 433/504] ITS#10198 Win32 mdb_strerror - stop passing "ignored"
 parameter

The M$ docs say the parameter is ignored, but it actually isn't,
and will cause a SEGV if the pointed memory isn't an init'd va_list.
---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index d1149fc556..2cc7d269e6 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1920,7 +1920,7 @@ mdb_strerror(int err)
 	buf[0] = 0;
 	FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM |
 		FORMAT_MESSAGE_IGNORE_INSERTS,
-		NULL, err, 0, ptr, MSGSIZE, (va_list *)buf+MSGSIZE);
+		NULL, err, 0, ptr, MSGSIZE, NULL);
 	return ptr;
 #else
 	return strerror(err);

From 4a25e0699bb2a58eaf390682ba72f745a19d86ae Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 2 May 2024 16:29:03 +0100
Subject: [PATCH 434/504] ITS#10212 LMDB: init txnid for read-only DBs

---
 libraries/liblmdb/mdb.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 2cc7d269e6..7dbfa59c18 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -3278,6 +3278,10 @@ mdb_txn_renew0(MDB_txn *txn)
 					UNLOCK_MUTEX(rmutex);
 					return MDB_READERS_FULL;
 				}
+				if ((env->me_flags & MDB_RDONLY) && !ti->mti_txnid)  {
+					meta = mdb_env_pick_meta(env);
+					ti->mti_txnid = meta->mm_txnid;
+				}
 				r = &ti->mti_readers[i];
 				/* Claim the reader slot, carefully since other code
 				 * uses the reader table un-mutexed: First reset the

From a6e8631ef92b0c6a8ce09e698483b69231ce16d0 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Fri, 3 May 2024 20:43:39 +0100
Subject: [PATCH 435/504] ITS#10212 LMDB: better fix

---
 libraries/liblmdb/mdb.c | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 7dbfa59c18..3914658b56 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -3278,10 +3278,6 @@ mdb_txn_renew0(MDB_txn *txn)
 					UNLOCK_MUTEX(rmutex);
 					return MDB_READERS_FULL;
 				}
-				if ((env->me_flags & MDB_RDONLY) && !ti->mti_txnid)  {
-					meta = mdb_env_pick_meta(env);
-					ti->mti_txnid = meta->mm_txnid;
-				}
 				r = &ti->mti_readers[i];
 				/* Claim the reader slot, carefully since other code
 				 * uses the reader table un-mutexed: First reset the
@@ -3307,9 +3303,14 @@ mdb_txn_renew0(MDB_txn *txn)
 			do /* LY: Retry on a race, ITS#7970. */
 				r->mr_txnid = ti->mti_txnid;
 			while(r->mr_txnid != ti->mti_txnid);
+			if (!r->mr_txnid && (env->me_flags & MDB_RDONLY)) {
+				meta = mdb_env_pick_meta(env);
+				r->mr_txnid = meta->mm_txnid;
+			} else {
+				meta = env->me_metas[r->mr_txnid & 1];
+			}
 			txn->mt_txnid = r->mr_txnid;
 			txn->mt_u.reader = r;
-			meta = env->me_metas[txn->mt_txnid & 1];
 		}
 
 	} else {

From 87dd0740e69dea944289568545c150a84b5f0e52 Mon Sep 17 00:00:00 2001
From: Zach Vonler <zach.vonler@sambanovasystems.com>
Date: Mon, 3 Jun 2024 12:39:02 -0700
Subject: [PATCH 436/504] ITS#10222 LMDB: Updated mdb_dump man page

The -a option to mdb_load makes the previous text obsolete.
---
 libraries/liblmdb/mdb_dump.1 | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/libraries/liblmdb/mdb_dump.1 b/libraries/liblmdb/mdb_dump.1
index acefe4e719..43b0b1c99a 100644
--- a/libraries/liblmdb/mdb_dump.1
+++ b/libraries/liblmdb/mdb_dump.1
@@ -80,14 +80,14 @@ Exit status is zero if no errors occur.
 Errors result in a non-zero exit status and
 a diagnostic message being written to standard error.
 
-Dumping and reloading databases that use user-defined comparison functions
-will result in new databases that use the default comparison functions.
-\fBIn this case it is quite likely that the reloaded database will be
-damaged beyond repair permitting neither record storage nor retrieval.\fP
+Dumping databases that use user-defined comparison functions will output
+records with the ordering imposed by those comparison functions. If
+.B mdb_load
+is invoked without including the
+.B -a
+option when reloading those records, the new databases will likely be
+damaged beyond repair, permitting neither record storage nor retrieval.
 
-The only available workaround is to modify the source for the
-.BR mdb_load (1)
-utility to load the database using the correct comparison functions.
 .SH "SEE ALSO"
 .BR mdb_load (1)
 .SH AUTHOR

From c883e8f712386d5e6bfc93ef18c66f1d5b605c20 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 3 Jun 2024 21:14:46 +0100
Subject: [PATCH 437/504] LMDB: tweak mdb_load.1 manpage

Add missing -a option to Synopsis
---
 libraries/liblmdb/mdb_load.1 | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/libraries/liblmdb/mdb_load.1 b/libraries/liblmdb/mdb_load.1
index aa25b03aad..bd02ea6070 100644
--- a/libraries/liblmdb/mdb_load.1
+++ b/libraries/liblmdb/mdb_load.1
@@ -8,6 +8,8 @@ mdb_load \- LMDB environment import tool
 [\c
 .BR \-V ]
 [\c
+.BR \-a ]
+[\c
 .BI \-f \ file\fR]
 [\c
 .BR \-n ]

From ce6f8e2d930a1a2d8db0bbb1a4dd80081772c891 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 17 Jun 2024 17:06:19 +0100
Subject: [PATCH 438/504] ITS#9378 don't try to print oversized keys in debug
 msg

---
 libraries/liblmdb/mdb.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 3914658b56..9ec7305efc 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1999,8 +1999,10 @@ mdb_dval(MDB_txn *txn, MDB_dbi dbi, MDB_val *data, char *buf)
 {
 	if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) {
 		mdb_dkey(data, buf+1);
-		*buf = '[';
-		strcpy(buf + data->mv_size * 2 + 1, "]");
+		if (data->mv_size < DKBUF_MAXKEYSIZE - 2) {
+			*buf = '[';
+			strcpy(buf + data->mv_size * 2 + 1, "]");
+		}
 	} else
 		*buf = '\0';
 	return buf;

From fd3c2adae70d2ed65017100db45e0b3babfe342a Mon Sep 17 00:00:00 2001
From: Andras <zanaviska@tutanota.com>
Date: Mon, 29 Jul 2024 14:28:16 +0300
Subject: [PATCH 439/504] ITS#10244 win32: Fix passed ptr type

From dfb3bbed656132456001c5aaca246fd4430e5ef5 ITS#9017
---
 libraries/liblmdb/mdb.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 9ec7305efc..43cb1aa01d 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -4213,9 +4213,11 @@ retry_seek:
 		rc = 0;
 		while (--async_i >= 0) {
 			if (ov[async_i].hEvent) {
-				if (!GetOverlappedResult(fd, &ov[async_i], &wres, TRUE)) {
+				DWORD temp_wres;
+				if (!GetOverlappedResult(fd, &ov[async_i], &temp_wres, TRUE)) {
 					rc = ErrCode(); /* Continue on so that all the event signals are reset */
 				}
+				wres = temp_wres;
 			}
 		}
 		if (rc) { /* any error on GetOverlappedResult, exit now */

From d61005822c120e5364dcb41893372d502a1bcc81 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 16 Sep 2024 21:11:08 +0100
Subject: [PATCH 440/504] ITS#9920 lmdb: fix page_split of encrypted page

---
 libraries/liblmdb/mdb.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 43cb1aa01d..32d5872ff1 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -10560,6 +10560,10 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 			copy->mp_flags = mp->mp_flags;
 			copy->mp_lower = (PAGEHDRSZ-PAGEBASE);
 			copy->mp_upper = env->me_psize - PAGEBASE;
+#if MDB_RPAGE_CACHE
+			copy->mp_upper -= env->me_sumsize;
+			copy->mp_upper -= env->me_esumsize;
+#endif
 
 			/* prepare to insert */
 			for (i=0, j=0; i<nkeys; i++) {

From 6cbb42aae0b0525dd5fb6f9d541848fd4d4bd59d Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 17 Sep 2024 00:11:22 +0100
Subject: [PATCH 441/504] ITS#9920 lmdb: cleanup prev commit

---
 libraries/liblmdb/mdb.c | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 32d5872ff1..2804ba250a 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -8970,11 +8970,7 @@ mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp)
 	    np->mp_pgno, mc->mc_txn->mt_env->me_psize));
 	np->mp_flags |= flags;
 	np->mp_lower = (PAGEHDRSZ-PAGEBASE);
-	np->mp_upper = mc->mc_txn->mt_env->me_psize - PAGEBASE;
-#if MDB_RPAGE_CACHE
-	np->mp_upper -= mc->mc_txn->mt_env->me_sumsize;
-	np->mp_upper -= mc->mc_txn->mt_env->me_esumsize;
-#endif
+	np->mp_upper = mc->mc_txn->mt_env->me_pagespace;
 
 	if (IS_BRANCH(np))
 		mc->mc_db->md_branch_pages++;
@@ -10559,11 +10555,7 @@ mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno
 			copy->mp_pgno  = mp->mp_pgno;
 			copy->mp_flags = mp->mp_flags;
 			copy->mp_lower = (PAGEHDRSZ-PAGEBASE);
-			copy->mp_upper = env->me_psize - PAGEBASE;
-#if MDB_RPAGE_CACHE
-			copy->mp_upper -= env->me_sumsize;
-			copy->mp_upper -= env->me_esumsize;
-#endif
+			copy->mp_upper = env->me_pagespace;
 
 			/* prepare to insert */
 			for (i=0, j=0; i<nkeys; i++) {

From acab3df034144e4792e5ac6c11e1317ac4581954 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 25 Sep 2024 18:23:31 +0100
Subject: [PATCH 442/504] LMDB: Add a separate mlm_setup0() to separate loading
 from setup

---
 libraries/liblmdb/module.c | 35 ++++++++++++++++++++---------------
 libraries/liblmdb/module.h |  1 +
 2 files changed, 21 insertions(+), 15 deletions(-)

diff --git a/libraries/liblmdb/module.c b/libraries/liblmdb/module.c
index 52a24eb019..16fefa96c0 100644
--- a/libraries/liblmdb/module.c
+++ b/libraries/liblmdb/module.c
@@ -76,26 +76,31 @@ void mlm_unload(void *mlm)
 #endif
 }
 
+void mlm_setup0(MDB_env *env, MDB_crypto_funcs *cf, const char *password)
+{
+	MDB_val enckey = {0};
+	if (cf->mcf_sumfunc) {
+		mdb_env_set_checksum(env, cf->mcf_sumfunc, cf->mcf_sumsize);
+	}
+	if (cf->mcf_encfunc && password) {
+		char keybuf[2048];
+		enckey.mv_data = keybuf;
+		enckey.mv_size = cf->mcf_keysize;
+		if (cf->mcf_str2key)
+			cf->mcf_str2key(password, &enckey);
+		else
+			strncpy(enckey.mv_data, password, enckey.mv_size);
+		mdb_env_set_encrypt(env, cf->mcf_encfunc, &enckey, cf->mcf_esumsize);
+		memset(enckey.mv_data, 0, enckey.mv_size);
+	}
+}
+
 void *mlm_setup(MDB_env *env, const char *file, const char *password, char **errmsg)
 {
 	MDB_crypto_funcs *cf;
-	MDB_val enckey = {0};
 	void *mlm = mlm_load(file, NULL, &cf, errmsg);
 	if (mlm) {
-		if (cf->mcf_sumfunc) {
-			mdb_env_set_checksum(env, cf->mcf_sumfunc, cf->mcf_sumsize);
-		}
-		if (cf->mcf_encfunc && password) {
-			char keybuf[2048];
-			enckey.mv_data = keybuf;
-			enckey.mv_size = cf->mcf_keysize;
-			if (cf->mcf_str2key)
-				cf->mcf_str2key(password, &enckey);
-			else
-				strncpy(enckey.mv_data, password, enckey.mv_size);
-			mdb_env_set_encrypt(env, cf->mcf_encfunc, &enckey, cf->mcf_esumsize);
-			memset(enckey.mv_data, 0, enckey.mv_size);
-		}
+		mlm_setup0(env, cf, password);
 	}
 	return mlm;
 }
diff --git a/libraries/liblmdb/module.h b/libraries/liblmdb/module.h
index 13d4494dd8..bd5fb3ff14 100644
--- a/libraries/liblmdb/module.h
+++ b/libraries/liblmdb/module.h
@@ -14,3 +14,4 @@
 void *mlm_load(const char *file, const char *name, MDB_crypto_funcs **mcf_ptr, char **errmsg);
 void mlm_unload(void *lm);
 void *mlm_setup(MDB_env *env, const char *file, const char *password, char **errmsg);
+void mlm_setup0(MDB_env *env, MDB_crypto_funcs *cf, const char *password);

From 49726aa38bd62497c3ef6f1f696c4ad751e76458 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 25 Sep 2024 20:05:15 +0100
Subject: [PATCH 443/504] Another fix for OpenSSL 3.0+

Should rewrite this module to use libsodium instead.
---
 libraries/liblmdb/crypto.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/crypto.c b/libraries/liblmdb/crypto.c
index fe98d3b979..a1e635d61d 100644
--- a/libraries/liblmdb/crypto.c
+++ b/libraries/liblmdb/crypto.c
@@ -77,6 +77,9 @@ typedef struct evp_cipher_head {
 	int key_len;
 	int iv_len;
 	unsigned long flags;
+#if OPENSSL_VERSION_NUMBER >= 0x30000000
+	int origin;
+#endif
 	int (*init)(EVP_CIPHER_CTX *ctx, const unsigned char *key,
 		const unsigned char *iv, int enc);
 } evp_cipher_head;
@@ -129,7 +132,7 @@ static int mcf_encfunc(const MDB_val *src, MDB_val *dst, const MDB_val *key, int
 	mdb_size_t *ptr;
 	MY_CIPHER_CTX myctx = {0};
 	EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)&myctx;
-	my_cipherdata cactx;
+	my_cipherdata cactx = {0};
 	evp_cipher_head *eh = (evp_cipher_head *)cipher;
 
 	ptr = key[1].mv_data;

From e895f5fd36b9bbfbb865baafb4a859cb942bed4b Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 14 Oct 2024 18:54:37 +0100
Subject: [PATCH 444/504] ITS#9920 crypto: use libsodium instead of OpenSSL

Note: even though the same algorithms were used, the data produced by
this module doesn't work with the OpenSSL-based module and vice versa.
---
 libraries/liblmdb/Makefile |   2 +-
 libraries/liblmdb/crypto.c | 165 +++++++------------------------------
 2 files changed, 30 insertions(+), 137 deletions(-)

diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile
index 7411b309f8..01d6536604 100644
--- a/libraries/liblmdb/Makefile
+++ b/libraries/liblmdb/Makefile
@@ -98,7 +98,7 @@ mtest_enc2:	  mtest_enc2.o module.o liblmdb.a crypto.lm
 mplay:	mplay.o liblmdb.a
 
 crypto.lm:	crypto.c
-	$(CC) -shared $(CFLAGS) -o $@ $^ -lcrypto
+	$(CC) -shared $(CFLAGS) -o $@ $^ -lsodium
 
 mdb.o: mdb.c lmdb.h midl.h
 	$(CC) $(CFLAGS) $(CPPFLAGS) -c mdb.c
diff --git a/libraries/liblmdb/crypto.c b/libraries/liblmdb/crypto.c
index a1e635d61d..14fc737c83 100644
--- a/libraries/liblmdb/crypto.c
+++ b/libraries/liblmdb/crypto.c
@@ -1,6 +1,6 @@
 /* crypto.c - LMDB encryption helper module */
 /*
- * Copyright 2020-2021 Howard Chu, Symas Corp.
+ * Copyright 2020-2024 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -12,173 +12,66 @@
  */
 #include <string.h>
 
-#include <openssl/engine.h>
+#include <sodium.h>
 
 #include "lmdb.h"
 
-MDB_crypto_hooks MDB_crypto;
+#define KEYBYTES	crypto_aead_chacha20poly1305_ietf_KEYBYTES
+#define NONCEBYTES	crypto_aead_chacha20poly1305_ietf_NPUBBYTES
+#define MACBYTES	crypto_aead_chacha20poly1305_ietf_ABYTES
 
-static EVP_CIPHER *cipher;
+MDB_crypto_hooks MDB_crypto;
 
 static int mcf_str2key(const char *passwd, MDB_val *key)
 {
-	unsigned int size;
-	EVP_MD_CTX *mdctx = EVP_MD_CTX_new();
-	EVP_DigestInit_ex(mdctx, EVP_sha256(), NULL);
-	EVP_DigestUpdate(mdctx, "Just a Constant", sizeof("Just a Constant"));
-	EVP_DigestUpdate(mdctx, passwd, strlen(passwd));
-	EVP_DigestFinal_ex(mdctx, key->mv_data, &size);
-	EVP_MD_CTX_free(mdctx);
+	crypto_hash_sha256_state state;
+
+	crypto_hash_sha256_init(&state);
+	crypto_hash_sha256_update(&state, (const unsigned char *)"Just a Constant", sizeof("Just a Constant"));
+	crypto_hash_sha256_update(&state, (const unsigned char *)passwd, strlen(passwd));
+	crypto_hash_sha256_final(&state, key->mv_data);
 	return 0;
 }
 
-/* cheats - internal OpenSSL 1.1 structures
- * These are copied from the OpenSSL source code.
- *
- * We use these to allow stack allocation of these structures
- * and to prevent OpenSSL from malloc'ing and free'ing them,
- * which would be too slow.
- */
-typedef struct my_cipher_ctx_st {
-    const EVP_CIPHER *cipher;
-    ENGINE *engine;             /* functional reference if 'cipher' is
-                                 * ENGINE-provided */
-    int encrypt;                /* encrypt or decrypt */
-    int buf_len;                /* number we have left */
-    unsigned char oiv[EVP_MAX_IV_LENGTH]; /* original iv */
-    unsigned char iv[EVP_MAX_IV_LENGTH]; /* working iv */
-    unsigned char buf[EVP_MAX_BLOCK_LENGTH]; /* saved partial block */
-    int num;                    /* used by cfb/ofb/ctr mode */
-    /* FIXME: Should this even exist? It appears unused */
-    void *app_data;             /* application stuff */
-    int key_len;                /* May change for variable length cipher */
-#if OPENSSL_VERSION_NUMBER >= 0x30006000
-    int iv_len;                 /* IV length */
-#endif
-    unsigned long flags;        /* Various flags */
-    void *cipher_data;          /* per EVP data */
-    int final_used;
-    int block_mask;
-    unsigned char final[EVP_MAX_BLOCK_LENGTH]; /* possible final block */
+#define ENC(dst, src, mac, key, iv) \
+	crypto_aead_chacha20poly1305_ietf_encrypt_detached(dst->mv_data, mac, &mlen, \
+		src->mv_data, src->mv_size, NULL, 0, NULL, iv, key)
 
-#if OPENSSL_VERSION_NUMBER >= 0x30000000
-    /*
-     * Opaque ctx returned from a providers cipher algorithm implementation
-     * OSSL_FUNC_cipher_newctx()
-     */
-    void *algctx;
-    EVP_CIPHER *fetched_cipher;
-#endif
-} MY_CIPHER_CTX;
-
-typedef struct evp_cipher_head {
-	int nid;
-	int block_size;
-	int key_len;
-	int iv_len;
-	unsigned long flags;
-#if OPENSSL_VERSION_NUMBER >= 0x30000000
-	int origin;
-#endif
-	int (*init)(EVP_CIPHER_CTX *ctx, const unsigned char *key,
-		const unsigned char *iv, int enc);
-} evp_cipher_head;
-
-#define	CHACHA_KEY_SIZE	32
-#define CHACHA_CTR_SIZE	16
-#define CHACHA_BLK_SIZE	64
-#define POLY1305_BLOCK_SIZE	16
-
-typedef struct {
-    union {
-        double align;   /* this ensures even sizeof(EVP_CHACHA_KEY)%8==0 */
-        unsigned int d[CHACHA_KEY_SIZE / 4];
-    } key;
-    unsigned int  counter[CHACHA_CTR_SIZE / 4];
-    unsigned char buf[CHACHA_BLK_SIZE];
-    unsigned int  partial_len;
-} MY_CHACHA_KEY;
-
-typedef struct {
-    MY_CHACHA_KEY key;
-    unsigned int nonce[12/4];
-    unsigned char tag[POLY1305_BLOCK_SIZE];
-    unsigned char tls_aad[POLY1305_BLOCK_SIZE];
-    struct { uint64_t aad, text; } len;
-    int aad, mac_inited, tag_len, nonce_len;
-    size_t tls_payload_length;
-} MY_CHACHA_AEAD_CTX;
-
-typedef struct {
-	double opaque[24];
-	unsigned int nonce[4];
-	unsigned char data[POLY1305_BLOCK_SIZE];
-	size_t num;
-	struct {
-		void (*foo1)();
-		void (*foo2)();
-	} func;
-} my_poly1305_ctx;
-
-typedef struct my_cipherdata {
-	MY_CHACHA_AEAD_CTX aead_ctx;
-	my_poly1305_ctx poly_ctx;
-} my_cipherdata;
+#define DEC(dst, src, mac, key, iv) \
+	crypto_aead_chacha20poly1305_ietf_decrypt_detached(dst->mv_data, NULL, src->mv_data, \
+		src->mv_size, mac, NULL, 0, iv, key)
 
 static int mcf_encfunc(const MDB_val *src, MDB_val *dst, const MDB_val *key, int encdec)
 {
-	unsigned char iv[12];
-	int ivl, outl, rc;
+	unsigned char iv[NONCEBYTES];
 	mdb_size_t *ptr;
-	MY_CIPHER_CTX myctx = {0};
-	EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)&myctx;
-	my_cipherdata cactx = {0};
-	evp_cipher_head *eh = (evp_cipher_head *)cipher;
+	int ivl, rc;
+	unsigned long long mlen;
 
+	/* the nonce is only 12 bytes */
 	ptr = key[1].mv_data;
 	ivl = ptr[0] & 0xffffffff;
 	memcpy(iv, &ivl, 4);
 	memcpy(iv+4, ptr+1, sizeof(mdb_size_t));
-	EVP_CipherInit_ex(ctx, cipher, NULL, NULL, NULL, encdec);
 
-	/* we can't set cipher_data before calling CipherInit because
-	 * that will just try to free it. So set it now, and then finish
-	 * up the other two Init calls that we disabled before.
-	 */
-	myctx.cipher_data = &cactx;
-	EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_INIT, 0, NULL);
-	eh->init(ctx, key[0].mv_data, iv, encdec);
-
-	EVP_CIPHER_CTX_set_padding(ctx, 0);
-	if (!encdec) {
-		EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_TAG, key[2].mv_size, key[2].mv_data);
+	if (encdec) {
+		rc = ENC(dst, src, key[2].mv_data, key[0].mv_data, iv);
+	} else {
+		rc = DEC(dst, src, key[2].mv_data, key[0].mv_data, iv);
 	}
-	rc = EVP_CipherUpdate(ctx, dst->mv_data, &outl, src->mv_data, src->mv_size);
-	if (rc)
-		rc = EVP_CipherFinal_ex(ctx, key[2].mv_data, &outl);
-	if (rc && encdec) {
-		EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_GET_TAG, key[2].mv_size, key[2].mv_data);
-	}
-	return rc == 0;
+	return rc;
 }
 
 static const MDB_crypto_funcs mcf_table = {
 	mcf_str2key,
 	mcf_encfunc,
 	NULL,
-	CHACHA_KEY_SIZE,
-	POLY1305_BLOCK_SIZE,
+	KEYBYTES,
+	MACBYTES,
 	0
 };
 
 MDB_crypto_funcs *MDB_crypto()
 {
-	evp_cipher_head *eh;
-	cipher = (EVP_CIPHER *)EVP_chacha20_poly1305();
-
-	/* We must disable the implicit init calls */
-	eh = (evp_cipher_head *)cipher;
-	eh->flags &= ~(EVP_CIPH_CTRL_INIT|EVP_CIPH_ALWAYS_CALL_INIT);
-
 	return (MDB_crypto_funcs *)&mcf_table;
 }

From c277aed15af9fe33e2c6b50a64043b6c3398af5a Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 14 Oct 2024 19:40:26 +0100
Subject: [PATCH 445/504] ITS#9920 lmdb: fix data races in rpage decryption

---
 libraries/liblmdb/mdb.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 2804ba250a..8863b6df5a 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -6810,7 +6810,11 @@ mdb_rpage_get(MDB_txn *txn, pgno_t pg0, int numpgs, MDB_page **ret)
 		id3.muse = tl[x].muse;
 		tl[x].mref++;
 		if (env->me_encfunc || env->me_sumfunc) {
+			if (env->me_encfunc)
+				pthread_mutex_lock(&env->me_rpmutex);
 			rc = mdb_rpage_encsum(env, &id3, rem, numpgs);
+			if (env->me_encfunc)
+				pthread_mutex_unlock(&env->me_rpmutex);
 			if (rc) return rc;
 			tl[x].muse = id3.muse;
 		}
@@ -6990,7 +6994,11 @@ ok:
 	p = (MDB_page *)(base + rem * env->me_psize);
 	rc = MDB_SUCCESS;
 	if (env->me_encfunc || env->me_sumfunc) {
+		if (env->me_encfunc)
+			pthread_mutex_lock(&env->me_rpmutex);
 		rc = mdb_rpage_encsum(env, &id3, rem, numpgs);
+		if (env->me_encfunc)
+			pthread_mutex_unlock(&env->me_rpmutex);
 	}
 #if MDB_DEBUG	/* we don't need this check any more */
 	if (IS_OVERFLOW(p)) {

From 1e7891c016a4518eaf0aa29f959b0cc8a22d4111 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 16 Oct 2024 17:36:18 +0100
Subject: [PATCH 446/504] ITS#9920 lmdb: fix page offset when converting
 subpage to real page

offset needs to account for additional space at page tail
---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 8863b6df5a..0dea5c63f4 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -8593,7 +8593,7 @@ prep_subDB:
 					if ((rc = mdb_page_alloc(mc, 1, &mp)))
 						return rc;
 					fp_flags |= mp->mp_flags; /* P_ADM_FLAGS */
-					offset = env->me_psize - olddata.mv_size;
+					offset = env->me_pagespace + PAGEHDRSZ - olddata.mv_size;
 					flags |= F_DUPDATA|F_SUBDATA;
 					dummy.md_root = mp->mp_pgno;
 					sub_root = mp;

From 14c7ff334759278d12db0aa5a8a3930a2cba3736 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 16 Oct 2024 22:47:42 +0100
Subject: [PATCH 447/504] Move the crypto module helpers into main liblmdb.

Just makes things easier than using the individual mdb_env_set APIs.
---
 libraries/liblmdb/Makefile     | 14 +++---
 libraries/liblmdb/lmdb.h       | 31 +++++++++++++
 libraries/liblmdb/mdb.c        | 82 ++++++++++++++++++++++++++++++++++
 libraries/liblmdb/mdb_copy.c   |  7 +--
 libraries/liblmdb/mdb_drop.c   |  7 +--
 libraries/liblmdb/mdb_dump.c   |  7 +--
 libraries/liblmdb/mdb_load.c   |  7 +--
 libraries/liblmdb/mdb_stat.c   |  7 +--
 libraries/liblmdb/mtest_enc2.c |  7 +--
 9 files changed, 144 insertions(+), 25 deletions(-)

diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile
index 01d6536604..e598afb8e2 100644
--- a/libraries/liblmdb/Makefile
+++ b/libraries/liblmdb/Makefile
@@ -74,15 +74,15 @@ liblmdb$(SOEXT):	mdb.lo midl.lo
 #	$(CC) $(LDFLAGS) -pthread -shared -Wl,-Bsymbolic -o $@ mdb.o midl.o $(SOLIBS)
 	$(CC) $(LDFLAGS) -pthread -shared -o $@ mdb.lo midl.lo $(SOLIBS)
 
-mdb_stat: mdb_stat.o module.o liblmdb.a
+mdb_stat: mdb_stat.o liblmdb.a
 	$(CC) $(LDFLAGS) -o $@ $^ $(LDL)
-mdb_copy: mdb_copy.o module.o liblmdb.a
+mdb_copy: mdb_copy.o liblmdb.a
 	$(CC) $(LDFLAGS) -o $@ $^ $(LDL)
-mdb_dump: mdb_dump.o module.o liblmdb.a
+mdb_dump: mdb_dump.o liblmdb.a
 	$(CC) $(LDFLAGS) -o $@ $^ $(LDL)
-mdb_load: mdb_load.o module.o liblmdb.a
+mdb_load: mdb_load.o liblmdb.a
 	$(CC) $(LDFLAGS) -o $@ $^ $(LDL)
-mdb_drop: mdb_drop.o module.o liblmdb.a
+mdb_drop: mdb_drop.o liblmdb.a
 	$(CC) $(LDFLAGS) -o $@ $^ $(LDL)
 mtest:    mtest.o    liblmdb.a
 mtest2:	mtest2.o liblmdb.a
@@ -92,8 +92,8 @@ mtest5:	mtest5.o liblmdb.a
 mtest6:	mtest6.o liblmdb.a
 mtest_remap:  mtest_remap.o liblmdb.a
 mtest_enc:    mtest_enc.o chacha8.o liblmdb.a
-mtest_enc2:	  mtest_enc2.o module.o liblmdb.a crypto.lm
-	$(CC) $(LDFLAGS) -pthread -o $@ mtest_enc2.o module.o liblmdb.a $(LDL)
+mtest_enc2:	  mtest_enc2.o liblmdb.a crypto.lm
+	$(CC) $(LDFLAGS) -pthread -o $@ mtest_enc2.o liblmdb.a $(LDL)
 
 mplay:	mplay.o liblmdb.a
 
diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 155bb2a3f6..4ab108116b 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -1760,6 +1760,37 @@ typedef struct MDB_crypto_funcs {
 	 * @return A pointer to a #MDB_crypto_funcs structure.
 	 */
 typedef MDB_crypto_funcs *(MDB_crypto_hooks)(void);
+
+	/** @brief Load a dynamically loadable module.
+	 *
+	 * @param[in] file The pathname of the module to load.
+	 * @param[in] symname The name of a symbol to resolve in the module.
+	 * @param[out] mcf_ptr The crypto hooks returned from the module.
+	 * @param[out] errmsg Messages for any errors from trying to load the module.
+	 * @return The handle to the loadable module that can be unloaded by #mdb_modunload(),
+	 * or NULL if loading failed.
+	 */
+void *mdb_modload(const char *file, const char *symname,
+			MDB_crypto_funcs **mcf_ptr, char **errmsg);
+
+	/** @brief Unload a dynamically loaded module.
+	 *
+	 * All environments that used the functions in the module must be closed
+	 * before unloading the module.
+	 * @param[in] handle The handle returned by #mdb_modload().
+	 */
+void mdb_modunload(void *handle);
+
+	/** @brief Set an environment to use the given crypto functions.
+	 *
+	 * This is just a wrapper around #mdb_env_set_encrypt() to ease use of
+	 * dynamically loaded crypto functions.
+	 * @param[in] env An environment handle returned by #mdb_env_create()
+	 * @param[in] funcs The crypto hooks retrieved by #mdb_modload().
+	 * @param[in] passphrase The secret used to generate the encryption key for the environment.
+	 */
+void mdb_modsetup(MDB_env *env, MDB_crypto_funcs *cf, const char *passphrase);
+
 /**	@} */
 
 #ifdef __cplusplus
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 0dea5c63f4..a5cada1e22 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -11510,6 +11510,88 @@ mdb_env_set_checksum(MDB_env *env, MDB_sum_func *func, unsigned int size)
 	env->me_sumsize = size;
 	return MDB_SUCCESS;
 }
+
+#ifdef _WIN32
+#include <windows.h>
+#else
+#include <dlfcn.h>
+#endif
+
+void * ESECT
+mdb_modload(const char *file, const char *name, MDB_crypto_funcs **mcf_ptr, char **errmsg)
+{
+	MDB_crypto_hooks *hookfunc;
+	void *ret = NULL;
+	if (!name)
+		name = "MDB_crypto";
+
+#ifdef _WIN32
+	{
+		HINSTANCE mlm = LoadLibrary(file);
+		if (mlm) {
+			hookfunc = GetProcAddress(mlm, name);
+			if (hookfunc)
+				*mcf_ptr = hookfunc();
+			else {
+				*errmsg = "Crypto hook function not found";
+				FreeLibrary(mlm);
+				mlm = NULL;
+			}
+		} else {
+			*errmsg = GetLastError();
+		}
+		ret = (void *)mlm;
+	}
+#else
+	{
+		void *mlm = dlopen(file, RTLD_NOW);
+		if (mlm) {
+			hookfunc = dlsym(mlm, name);
+			if (hookfunc)
+				*mcf_ptr = hookfunc();
+			else {
+				*errmsg = "Crypto hook function not found";
+				dlclose(mlm);
+				mlm = NULL;
+			}
+		} else {
+			*errmsg = dlerror();
+		}
+		ret = mlm;
+	}
+#endif
+	return ret;
+}
+
+void ESECT
+mdb_modunload(void *mlm)
+{
+#ifdef _WIN32
+	FreeLibrary((HINSTANCE)mlm);
+#else
+	dlclose(mlm);
+#endif
+}
+
+void ESECT
+mdb_modsetup(MDB_env *env, MDB_crypto_funcs *cf, const char *password)
+{
+	MDB_val enckey = {0};
+	if (cf->mcf_sumfunc) {
+		mdb_env_set_checksum(env, cf->mcf_sumfunc, cf->mcf_sumsize);
+	}
+	if (cf->mcf_encfunc && password) {
+		char keybuf[2048];
+		enckey.mv_data = keybuf;
+		enckey.mv_size = cf->mcf_keysize;
+		if (cf->mcf_str2key)
+			cf->mcf_str2key(password, &enckey);
+		else
+			strncpy(enckey.mv_data, password, enckey.mv_size);
+		mdb_env_set_encrypt(env, cf->mcf_encfunc, &enckey, cf->mcf_esumsize);
+		memset(enckey.mv_data, 0, enckey.mv_size);
+	}
+}
 #endif
 
 int ESECT
diff --git a/libraries/liblmdb/mdb_copy.c b/libraries/liblmdb/mdb_copy.c
index c230caa2cb..522d315cf9 100644
--- a/libraries/liblmdb/mdb_copy.c
+++ b/libraries/liblmdb/mdb_copy.c
@@ -21,7 +21,6 @@
 #include <stdlib.h>
 #include <signal.h>
 #include "lmdb.h"
-#include "module.h"
 
 static void
 sighandle(int sig)
@@ -80,11 +79,13 @@ int main(int argc,char * argv[])
 	rc = mdb_env_create(&env);
 	if (rc == MDB_SUCCESS) {
 		if (module) {
-			mlm = mlm_setup(env, module, password, &errmsg);
+			MDB_crypto_funcs *mcf;
+			mlm = mdb_modload(module, NULL, &mcf, &errmsg);
 			if (!mlm) {
 				fprintf(stderr, "Failed to load crypto module: %s\n", errmsg);
 				exit(EXIT_FAILURE);
 			}
+			mdb_modsetup(env, mcf, password);
 		}
 		rc = mdb_env_open(env, argv[1], flags, 0600);
 	}
@@ -100,7 +101,7 @@ int main(int argc,char * argv[])
 			progname, act, rc, mdb_strerror(rc));
 	mdb_env_close(env);
 	if (mlm)
-		mlm_unload(mlm);
+		mdb_modunload(mlm);
 
 	return rc ? EXIT_FAILURE : EXIT_SUCCESS;
 }
diff --git a/libraries/liblmdb/mdb_drop.c b/libraries/liblmdb/mdb_drop.c
index 54e91332f0..2707e7b0f4 100644
--- a/libraries/liblmdb/mdb_drop.c
+++ b/libraries/liblmdb/mdb_drop.c
@@ -19,7 +19,6 @@
 #include <unistd.h>
 #include <signal.h>
 #include "lmdb.h"
-#include "module.h"
 
 static volatile sig_atomic_t gotsig;
 
@@ -103,11 +102,13 @@ int main(int argc, char *argv[])
 		return EXIT_FAILURE;
 	}
 	if (module) {
-		mlm = mlm_setup(env, module, password, &errmsg);
+		MDB_crypto_funcs *mcf;
+		mlm = mdb_modload(module, NULL, &mcf, &errmsg);
 		if (!mlm) {
 			fprintf(stderr, "Failed to load crypto module: %s\n", errmsg);
 			goto env_close;
 		}
+		mdb_modsetup(env, mcf, password);
 	}
 
 	mdb_env_set_maxdbs(env, 2);
@@ -148,7 +149,7 @@ txn_abort:
 env_close:
 	mdb_env_close(env);
 	if (mlm)
-		mlm_unload(mlm);
+		mdb_modunload(mlm);
 
 	return rc ? EXIT_FAILURE : EXIT_SUCCESS;
 }
diff --git a/libraries/liblmdb/mdb_dump.c b/libraries/liblmdb/mdb_dump.c
index a3af117e02..8ccd04833d 100644
--- a/libraries/liblmdb/mdb_dump.c
+++ b/libraries/liblmdb/mdb_dump.c
@@ -19,7 +19,6 @@
 #include <unistd.h>
 #include <signal.h>
 #include "lmdb.h"
-#include "module.h"
 
 #define Yu	MDB_PRIy(u)
 
@@ -250,11 +249,13 @@ int main(int argc, char *argv[])
 	}
 
 	if (module) {
-		mlm = mlm_setup(env, module, password, &errmsg);
+		MDB_crypto_funcs *mcf;
+		mlm = mdb_modload(module, NULL, &mcf, &errmsg);
 		if (!mlm) {
 			fprintf(stderr, "Failed to load crypto module: %s\n", errmsg);
 			goto env_close;
 		}
+		mdb_modsetup(env, mcf, password);
 	}
 
 	if (alldbs || subname) {
@@ -327,7 +328,7 @@ txn_abort:
 env_close:
 	mdb_env_close(env);
 	if (mlm)
-		mlm_unload(mlm);
+		mdb_modunload(mlm);
 
 	return rc ? EXIT_FAILURE : EXIT_SUCCESS;
 }
diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c
index 19dd272c57..337c12463a 100644
--- a/libraries/liblmdb/mdb_load.c
+++ b/libraries/liblmdb/mdb_load.c
@@ -18,7 +18,6 @@
 #include <ctype.h>
 #include <unistd.h>
 #include "lmdb.h"
-#include "module.h"
 
 #define PRINT	1
 #define NOHDR	2
@@ -382,11 +381,13 @@ int main(int argc, char *argv[])
 		return EXIT_FAILURE;
 	}
 	if (module) {
-		mlm = mlm_setup(env, module, password, &errmsg);
+		MDB_crypto_funcs *mcf;
+		mlm = mdb_modload(module, NULL, &mcf, &errmsg);
 		if (!mlm) {
 			fprintf(stderr, "Failed to load crypto module: %s\n", errmsg);
 			goto env_close;
 		}
+		mdb_modsetup(env, mcf, password);
 	}
 
 	mdb_env_set_maxdbs(env, 2);
@@ -526,7 +527,7 @@ txn_abort:
 env_close:
 	mdb_env_close(env);
 	if (mlm)
-		mlm_unload(mlm);
+		mdb_modunload(mlm);
 
 	return rc ? EXIT_FAILURE : EXIT_SUCCESS;
 }
diff --git a/libraries/liblmdb/mdb_stat.c b/libraries/liblmdb/mdb_stat.c
index caf519b341..9ae31861b8 100644
--- a/libraries/liblmdb/mdb_stat.c
+++ b/libraries/liblmdb/mdb_stat.c
@@ -16,7 +16,6 @@
 #include <string.h>
 #include <unistd.h>
 #include "lmdb.h"
-#include "module.h"
 
 #define Z	MDB_FMT_Z
 #define Yu	MDB_PRIy(u)
@@ -119,11 +118,13 @@ int main(int argc, char *argv[])
 	}
 
 	if (module) {
-		mlm = mlm_setup(env, module, password, &errmsg);
+		MDB_crypto_funcs *mcf;
+		mlm = mdb_modload(module, NULL, &mcf, &errmsg);
 		if (!mlm) {
 			fprintf(stderr, "Failed to load crypto module: %s\n", errmsg);
 			goto env_close;
 		}
+		mdb_modsetup(env, mcf, password);
 	}
 
 	if (alldbs || subname) {
@@ -270,7 +271,7 @@ txn_abort:
 env_close:
 	mdb_env_close(env);
 	if (mlm)
-		mlm_unload(mlm);
+		mdb_modunload(mlm);
 
 	return rc ? EXIT_FAILURE : EXIT_SUCCESS;
 }
diff --git a/libraries/liblmdb/mtest_enc2.c b/libraries/liblmdb/mtest_enc2.c
index 853960ab52..ea144e3f0f 100644
--- a/libraries/liblmdb/mtest_enc2.c
+++ b/libraries/liblmdb/mtest_enc2.c
@@ -15,7 +15,6 @@
 #include <stdlib.h>
 #include <time.h>
 #include "lmdb.h"
-#include "module.h"
 
 #define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr)
 #define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0))
@@ -40,6 +39,7 @@ int main(int argc,char * argv[])
 	char password[] = "This is my passphrase for now...";
 	void *mlm;
 	char *errmsg;
+	MDB_crypto_funcs *mcf;
 
 	srand(time(NULL));
 
@@ -51,11 +51,12 @@ int main(int argc,char * argv[])
 	    }
     
 		E(mdb_env_create(&env));
-		mlm = mlm_setup(env, "./crypto.lm", password, &errmsg);
+		mlm = mdb_modload("./crypto.lm", NULL, &mcf, &errmsg);
 		if (!mlm) {
 			fprintf(stderr,"Failed to load crypto module: %s\n", errmsg);
 			exit(1);
 		}
+		mdb_modsetup(env, mcf, password);
 		E(mdb_env_set_maxreaders(env, 1));
 		E(mdb_env_set_mapsize(env, 10485760));
 		E(mdb_env_open(env, "./testdb", 0 /*|MDB_NOSYNC*/, 0664));
@@ -183,7 +184,7 @@ int main(int argc,char * argv[])
 
 		mdb_dbi_close(env, dbi);
 		mdb_env_close(env);
-		mlm_unload(mlm);
+		mdb_modunload(mlm);
 
 	return 0;
 }

From b3fc03bae6bc44fafdf634c11896d3e19bc1b8d9 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 16 Oct 2024 22:50:06 +0100
Subject: [PATCH 448/504] Delete unused module code

---
 libraries/liblmdb/module.c | 106 -------------------------------------
 libraries/liblmdb/module.h |  17 ------
 2 files changed, 123 deletions(-)
 delete mode 100644 libraries/liblmdb/module.c
 delete mode 100644 libraries/liblmdb/module.h

diff --git a/libraries/liblmdb/module.c b/libraries/liblmdb/module.c
deleted file mode 100644
index 16fefa96c0..0000000000
--- a/libraries/liblmdb/module.c
+++ /dev/null
@@ -1,106 +0,0 @@
-/* module.c - helper for dynamically loading crypto module */
-/*
- * Copyright 2020-2021 Howard Chu, Symas Corp.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the Symas
- * Dual-Use License.
- *
- * A copy of this license is available in the file LICENSE in the
- * source distribution.
- */
-#ifdef _WIN32
-#include <windows.h>
-#else
-#include <dlfcn.h>
-#endif
-
-#include <stddef.h>
-#include <string.h>
-
-#include "lmdb.h"
-#include "module.h"
-
-void *mlm_load(const char *file, const char *name, MDB_crypto_funcs **mcf_ptr, char **errmsg)
-{
-	MDB_crypto_hooks *hookfunc;
-	void *ret = NULL;
-	if (!name)
-		name = "MDB_crypto";
-
-#ifdef _WIN32
-	{
-		HINSTANCE mlm = LoadLibrary(file);
-		if (mlm) {
-			hookfunc = GetProcAddress(mlm, name);
-			if (hookfunc)
-				*mcf_ptr = hookfunc();
-			else {
-				*errmsg = "Crypto hook function not found";
-				FreeLibrary(mlm);
-				mlm = NULL;
-			}
-		} else {
-			*errmsg = GetLastError();
-		}
-		ret = (void *)mlm;
-	}
-#else
-	{
-		void *mlm = dlopen(file, RTLD_NOW);
-		if (mlm) {
-			hookfunc = dlsym(mlm, name);
-			if (hookfunc)
-				*mcf_ptr = hookfunc();
-			else {
-				*errmsg = "Crypto hook function not found";
-				dlclose(mlm);
-				mlm = NULL;
-			}
-		} else {
-			*errmsg = dlerror();
-		}
-		ret = mlm;
-	}
-#endif
-	return ret;
-}
-
-void mlm_unload(void *mlm)
-{
-#ifdef _WIN32
-	FreeLibrary((HINSTANCE)mlm);
-#else
-	dlclose(mlm);
-#endif
-}
-
-void mlm_setup0(MDB_env *env, MDB_crypto_funcs *cf, const char *password)
-{
-	MDB_val enckey = {0};
-	if (cf->mcf_sumfunc) {
-		mdb_env_set_checksum(env, cf->mcf_sumfunc, cf->mcf_sumsize);
-	}
-	if (cf->mcf_encfunc && password) {
-		char keybuf[2048];
-		enckey.mv_data = keybuf;
-		enckey.mv_size = cf->mcf_keysize;
-		if (cf->mcf_str2key)
-			cf->mcf_str2key(password, &enckey);
-		else
-			strncpy(enckey.mv_data, password, enckey.mv_size);
-		mdb_env_set_encrypt(env, cf->mcf_encfunc, &enckey, cf->mcf_esumsize);
-		memset(enckey.mv_data, 0, enckey.mv_size);
-	}
-}
-
-void *mlm_setup(MDB_env *env, const char *file, const char *password, char **errmsg)
-{
-	MDB_crypto_funcs *cf;
-	void *mlm = mlm_load(file, NULL, &cf, errmsg);
-	if (mlm) {
-		mlm_setup0(env, cf, password);
-	}
-	return mlm;
-}
diff --git a/libraries/liblmdb/module.h b/libraries/liblmdb/module.h
deleted file mode 100644
index bd5fb3ff14..0000000000
--- a/libraries/liblmdb/module.h
+++ /dev/null
@@ -1,17 +0,0 @@
-/* module.h - helper for dynamically loading crypto module */
-/*
- * Copyright 2020-2021 Howard Chu, Symas Corp.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted only as authorized by the Symas
- * Dual-Use License.
- *
- * A copy of this license is available in the file LICENSE in the
- * source distribution.
- */
-
-void *mlm_load(const char *file, const char *name, MDB_crypto_funcs **mcf_ptr, char **errmsg);
-void mlm_unload(void *lm);
-void *mlm_setup(MDB_env *env, const char *file, const char *password, char **errmsg);
-void mlm_setup0(MDB_env *env, MDB_crypto_funcs *cf, const char *password);

From 25635d8251b2710e320e5d796ed922c2d88dd038 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 16 Oct 2024 23:11:20 +0100
Subject: [PATCH 449/504] Cleanup redundant include

---
 libraries/liblmdb/mdb.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index a5cada1e22..2580a92a4a 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -11511,9 +11511,7 @@ mdb_env_set_checksum(MDB_env *env, MDB_sum_func *func, unsigned int size)
 	return MDB_SUCCESS;
 }
 
-#ifdef _WIN32
-#include <windows.h>
-#else
+#ifndef _WIN32
 #include <dlfcn.h>
 #endif
 

From defcb167fb9d050993c66e94b8eb834418dd135a Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 16 Oct 2024 23:28:02 +0100
Subject: [PATCH 450/504] Keep module code in a separate object file

That way programs that don't use these functions won't needlessly
depend on -ldl
---
 libraries/liblmdb/Makefile | 11 +++--
 libraries/liblmdb/mdb.c    | 80 -------------------------------
 libraries/liblmdb/module.c | 98 ++++++++++++++++++++++++++++++++++++++
 3 files changed, 105 insertions(+), 84 deletions(-)
 create mode 100644 libraries/liblmdb/module.c

diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile
index e598afb8e2..e0c0532159 100644
--- a/libraries/liblmdb/Makefile
+++ b/libraries/liblmdb/Makefile
@@ -67,12 +67,12 @@ test:	all
 	rm -rf testdb && mkdir testdb
 	./mtest && ./mdb_stat testdb
 
-liblmdb.a:	mdb.o midl.o
-	$(AR) rs $@ mdb.o midl.o
+liblmdb.a:	mdb.o midl.o module.o
+	$(AR) rs $@ mdb.o midl.o module.o
 
-liblmdb$(SOEXT):	mdb.lo midl.lo
+liblmdb$(SOEXT):	mdb.lo midl.lo module.lo
 #	$(CC) $(LDFLAGS) -pthread -shared -Wl,-Bsymbolic -o $@ mdb.o midl.o $(SOLIBS)
-	$(CC) $(LDFLAGS) -pthread -shared -o $@ mdb.lo midl.lo $(SOLIBS)
+	$(CC) $(LDFLAGS) -pthread -shared -o $@ mdb.lo midl.lo module.lo $(SOLIBS) $(LDL)
 
 mdb_stat: mdb_stat.o liblmdb.a
 	$(CC) $(LDFLAGS) -o $@ $^ $(LDL)
@@ -112,6 +112,9 @@ mdb.lo: mdb.c lmdb.h midl.h
 midl.lo: midl.c midl.h
 	$(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -c midl.c -o $@
 
+module.lo: module.c lmdb.h
+	$(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -c module.c -o $@
+
 %:	%.o
 	$(CC) $(CFLAGS) $(LDFLAGS) $^ $(LDLIBS) -o $@
 
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 2580a92a4a..0dea5c63f4 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -11510,86 +11510,6 @@ mdb_env_set_checksum(MDB_env *env, MDB_sum_func *func, unsigned int size)
 	env->me_sumsize = size;
 	return MDB_SUCCESS;
 }
-
-#ifndef _WIN32
-#include <dlfcn.h>
-#endif
-
-void * ESECT
-mdb_modload(const char *file, const char *name, MDB_crypto_funcs **mcf_ptr, char **errmsg)
-{
-	MDB_crypto_hooks *hookfunc;
-	void *ret = NULL;
-	if (!name)
-		name = "MDB_crypto";
-
-#ifdef _WIN32
-	{
-		HINSTANCE mlm = LoadLibrary(file);
-		if (mlm) {
-			hookfunc = GetProcAddress(mlm, name);
-			if (hookfunc)
-				*mcf_ptr = hookfunc();
-			else {
-				*errmsg = "Crypto hook function not found";
-				FreeLibrary(mlm);
-				mlm = NULL;
-			}
-		} else {
-			*errmsg = GetLastError();
-		}
-		ret = (void *)mlm;
-	}
-#else
-	{
-		void *mlm = dlopen(file, RTLD_NOW);
-		if (mlm) {
-			hookfunc = dlsym(mlm, name);
-			if (hookfunc)
-				*mcf_ptr = hookfunc();
-			else {
-				*errmsg = "Crypto hook function not found";
-				dlclose(mlm);
-				mlm = NULL;
-			}
-		} else {
-			*errmsg = dlerror();
-		}
-		ret = mlm;
-	}
-#endif
-	return ret;
-}
-
-void ESECT
-mdb_modunload(void *mlm)
-{
-#ifdef _WIN32
-	FreeLibrary((HINSTANCE)mlm);
-#else
-	dlclose(mlm);
-#endif
-}
-
-void ESECT
-mdb_modsetup(MDB_env *env, MDB_crypto_funcs *cf, const char *password)
-{
-	MDB_val enckey = {0};
-	if (cf->mcf_sumfunc) {
-		mdb_env_set_checksum(env, cf->mcf_sumfunc, cf->mcf_sumsize);
-	}
-	if (cf->mcf_encfunc && password) {
-		char keybuf[2048];
-		enckey.mv_data = keybuf;
-		enckey.mv_size = cf->mcf_keysize;
-		if (cf->mcf_str2key)
-			cf->mcf_str2key(password, &enckey);
-		else
-			strncpy(enckey.mv_data, password, enckey.mv_size);
-		mdb_env_set_encrypt(env, cf->mcf_encfunc, &enckey, cf->mcf_esumsize);
-		memset(enckey.mv_data, 0, enckey.mv_size);
-	}
-}
 #endif
 
 int ESECT
diff --git a/libraries/liblmdb/module.c b/libraries/liblmdb/module.c
new file mode 100644
index 0000000000..63062ad881
--- /dev/null
+++ b/libraries/liblmdb/module.c
@@ -0,0 +1,98 @@
+/* module.c - helper for dynamically loading crypto module */
+/*
+ * Copyright 2020-2021 Howard Chu, Symas Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the Symas
+ * Dual-Use License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * source distribution.
+ */
+#ifdef _WIN32
+#include <windows.h>
+#else
+#include <dlfcn.h>
+#endif
+
+#include <stddef.h>
+#include <string.h>
+
+#include "lmdb.h"
+
+void *
+mdb_modload(const char *file, const char *name, MDB_crypto_funcs **mcf_ptr, char **errmsg)
+{
+	MDB_crypto_hooks *hookfunc;
+	void *ret = NULL;
+	if (!name)
+		name = "MDB_crypto";
+
+#ifdef _WIN32
+	{
+		HINSTANCE mlm = LoadLibrary(file);
+		if (mlm) {
+			hookfunc = GetProcAddress(mlm, name);
+			if (hookfunc)
+				*mcf_ptr = hookfunc();
+			else {
+				*errmsg = "Crypto hook function not found";
+				FreeLibrary(mlm);
+				mlm = NULL;
+			}
+		} else {
+			*errmsg = GetLastError();
+		}
+		ret = (void *)mlm;
+	}
+#else
+	{
+		void *mlm = dlopen(file, RTLD_NOW);
+		if (mlm) {
+			hookfunc = dlsym(mlm, name);
+			if (hookfunc)
+				*mcf_ptr = hookfunc();
+			else {
+				*errmsg = "Crypto hook function not found";
+				dlclose(mlm);
+				mlm = NULL;
+			}
+		} else {
+			*errmsg = dlerror();
+		}
+		ret = mlm;
+	}
+#endif
+	return ret;
+}
+
+void
+mdb_modunload(void *mlm)
+{
+#ifdef _WIN32
+	FreeLibrary((HINSTANCE)mlm);
+#else
+	dlclose(mlm);
+#endif
+}
+
+void
+mdb_modsetup(MDB_env *env, MDB_crypto_funcs *cf, const char *password)
+{
+	MDB_val enckey = {0};
+	if (cf->mcf_sumfunc) {
+		mdb_env_set_checksum(env, cf->mcf_sumfunc, cf->mcf_sumsize);
+	}
+	if (cf->mcf_encfunc && password) {
+		char keybuf[2048];
+		enckey.mv_data = keybuf;
+		enckey.mv_size = cf->mcf_keysize;
+		if (cf->mcf_str2key)
+			cf->mcf_str2key(password, &enckey);
+		else
+			strncpy(enckey.mv_data, password, enckey.mv_size);
+		mdb_env_set_encrypt(env, cf->mcf_encfunc, &enckey, cf->mcf_esumsize);
+		memset(enckey.mv_data, 0, enckey.mv_size);
+	}
+}

From 63091b38e46cb64c954feb510307b325ed7bbd25 Mon Sep 17 00:00:00 2001
From: Gary Wicker <gkwicker@amazon.com>
Date: Fri, 25 Oct 2024 01:05:00 +0000
Subject: [PATCH 451/504] ITS#10275 mdb_load: add -Q option to use NOSYNC for
 faster loading

---
 libraries/liblmdb/mdb_load.1 |  5 +++++
 libraries/liblmdb/mdb_load.c | 13 ++++++++++++-
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb_load.1 b/libraries/liblmdb/mdb_load.1
index bd02ea6070..c6f9554e26 100644
--- a/libraries/liblmdb/mdb_load.1
+++ b/libraries/liblmdb/mdb_load.1
@@ -22,6 +22,8 @@ mdb_load \- LMDB environment import tool
 [\c
 .BR \-N ]
 [\c
+.BR \-Q ]
+[\c
 .BR \-T ]
 .BR \ envpath
 .SH DESCRIPTION
@@ -71,6 +73,9 @@ Load a specific subdatabase. If no database is specified, data is loaded into th
 .BR \-N
 Don't overwrite existing records when loading into an already existing database; just skip them.
 .TP
+.BR \-Q
+Quick mode, uses MDB_NOSYNC for faster loading.  Forces sync with mdb_env_sync() before exiting.
+.TP
 .BR \-T
 Load data from simple text files. The input must be paired lines of text, where the first
 line of the pair is the key item, and the second line of the pair is its corresponding
diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c
index 337c12463a..6d1b5a4328 100644
--- a/libraries/liblmdb/mdb_load.c
+++ b/libraries/liblmdb/mdb_load.c
@@ -323,10 +323,11 @@ int main(int argc, char *argv[])
 	 * -n: use NOSUBDIR flag on env_open
 	 * -s: load into named subDB
 	 * -N: use NOOVERWRITE on puts
+	 * -Q: quick mode using NOSYNC
 	 * -T: read plaintext
 	 * -V: print version and exit
 	 */
-	while ((i = getopt(argc, argv, "af:m:ns:w:NTV")) != EOF) {
+	while ((i = getopt(argc, argv, "af:m:ns:w:NQTV")) != EOF) {
 		switch(i) {
 		case 'V':
 			printf("%s\n", MDB_VERSION_STRING);
@@ -351,6 +352,9 @@ int main(int argc, char *argv[])
 		case 'N':
 			putflags = MDB_NOOVERWRITE|MDB_NODUPDATA;
 			break;
+		case 'Q':
+			envflags |= MDB_NOSYNC;
+			break;
 		case 'T':
 			mode |= NOHDR | PRINT;
 			break;
@@ -519,6 +523,13 @@ int main(int argc, char *argv[])
 				prog, lineno, mdb_strerror(rc));
 			goto env_close;
 		}
+		if (envflags & MDB_NOSYNC) {
+			rc = mdb_env_sync(env, 1);
+			if (rc) {
+				fprintf(stderr, "mdb_env_sync failed, error %d %s\n", rc, mdb_strerror(rc));
+				goto env_close;
+			}
+		}
 		mdb_dbi_close(env, dbi);
 	}
 

From fc757af206f965d6b281e384441d3a7d93f9d017 Mon Sep 17 00:00:00 2001
From: kero <renault.cle@gmail.com>
Date: Thu, 9 Jan 2025 14:56:00 +0000
Subject: [PATCH 452/504] ITS#10296 lmdb: fix fdatasync on MacOS

Patch from kero, with corrections by hyc
---
 libraries/liblmdb/mdb.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 0dea5c63f4..da62da718e 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -170,7 +170,11 @@ typedef SSIZE_T	ssize_t;
 # if !(defined(MDB_USE_POSIX_MUTEX) || defined(MDB_USE_POSIX_SEM))
 # define MDB_USE_SYSV_SEM	1
 # endif
+# if defined(__APPLE__)
+# define MDB_FDATASYNC(fd)		fcntl(fd, F_FULLSYNC)
+# else
 # define MDB_FDATASYNC		fsync
+# endif
 #elif defined(__ANDROID__)
 # define MDB_FDATASYNC		fsync
 #endif
@@ -3075,7 +3079,7 @@ mdb_env_sync0(MDB_env *env, int force, pgno_t numpgs)
 				? MS_ASYNC : MS_SYNC;
 			if (MDB_MSYNC(env->me_map, env->me_psize * numpgs, flags))
 				rc = ErrCode();
-#ifdef _WIN32
+#if defined(_WIN32) || defined(__APPLE__)
 			else if (flags == MS_SYNC && MDB_FDATASYNC(env->me_fd))
 				rc = ErrCode();
 #endif

From 9ec98ca00e8689de4fcd5400f2d5722cf14c56e8 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 19 Feb 2025 09:13:46 +0000
Subject: [PATCH 453/504] ITS#10296 lmdb: fix typo in prev commit

---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index da62da718e..4346c5b89f 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -171,7 +171,7 @@ typedef SSIZE_T	ssize_t;
 # define MDB_USE_SYSV_SEM	1
 # endif
 # if defined(__APPLE__)
-# define MDB_FDATASYNC(fd)		fcntl(fd, F_FULLSYNC)
+# define MDB_FDATASYNC(fd)		fcntl(fd, F_FULLFSYNC)
 # else
 # define MDB_FDATASYNC		fsync
 # endif

From b6c4293f6fd6e3a3e4d5b6571fe3da7e80699597 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Fri, 21 Feb 2025 04:42:03 +0000
Subject: [PATCH 454/504] ITS#10024 lmdb: fix MDB_PREVSNAPSHOT txnid
 initialization

---
 libraries/liblmdb/mdb.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 4346c5b89f..dd00498322 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5446,6 +5446,9 @@ mdb_env_open2(MDB_env *env, int prev)
 #endif
 	env->me_maxpg = env->me_mapsize / env->me_psize;
 
+	if (prev && env->me_txns)
+		env->me_txns->mti_txnid = meta.mm_txnid;
+
 #if MDB_DEBUG
 	{
 		MDB_meta *meta = mdb_env_pick_meta(env);

From 71fe8350561088a3e2f5ed585a924d7326228911 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 7 Apr 2025 16:46:52 +0100
Subject: [PATCH 455/504] More for rawpart

It was ignoring the mapsize recorded in the meta page, and was
redundantly mapping the meta pages.
---
 libraries/liblmdb/mdb.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index dd00498322..1aa721ba26 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5373,6 +5373,14 @@ mdb_env_open2(MDB_env *env, int prev)
 #endif
 	;
 
+	/* if RAWPART, we did a preliminary mapping in read_header. drop it now. */
+	if (flags & MDB_RAWPART) {
+		munmap(env->me_map, env->me_mapsize);
+		env->me_map = NULL;
+		/* if mapsize was defaulted, use mapsize from metapage */
+		if (env->me_mapsize == DEFAULT_MAPSIZE)
+			env->me_mapsize = 0;
+	}
 	/* Was a mapsize configured? */
 	if (!env->me_mapsize) {
 		env->me_mapsize = meta.mm_mapsize;

From 2b77f6ffc24a06225c7ca67cd32e1270764061e4 Mon Sep 17 00:00:00 2001
From: jinyaoguo <guo846@purdue.edu>
Date: Tue, 27 May 2025 21:47:48 -0400
Subject: [PATCH 456/504] ITS#10342 lmdb: fix potential memleak in child
 txn_begin

---
 libraries/liblmdb/mdb.c | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 1aa721ba26..eaf3a2d99b 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -3547,10 +3547,8 @@ renew:
 	}
 	if (rc) {
 		if (txn != env->me_txn0) {
-#if MDB_RPAGE_CACHE
-			if (MDB_REMAPPING(env->me_flags))
-				free(txn->mt_rpages);
-#endif
+			/* mt_rpages belongs to parent */
+			free(txn->mt_u.dirty_list);
 			free(txn);
 		}
 	} else {

From b4aeb8cfd5b47e01243371662ddbc5105b9d761a Mon Sep 17 00:00:00 2001
From: Mike Moritz <mike.moritz@vertex.link>
Date: Wed, 28 May 2025 23:11:38 +0000
Subject: [PATCH 457/504] ITS#10346 lmdb: fix compacting copy with large values

---
 libraries/liblmdb/mdb.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index eaf3a2d99b..b8746bc6c4 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -10895,8 +10895,8 @@ typedef struct mdb_copy {
 	pthread_cond_t mc_cond;	/**< Condition variable for #mc_new */
 	char *mc_wbuf[2];
 	char *mc_over[2];
-	int mc_wlen[2];
-	int mc_olen[2];
+	size_t mc_wlen[2];
+	size_t mc_olen[2];
 	pgno_t mc_next_pgno;
 	HANDLE mc_fd;
 	int mc_toggle;			/**< Buffer number in provider */
@@ -10913,7 +10913,8 @@ mdb_env_copythr(void *arg)
 {
 	mdb_copy *my = arg;
 	char *ptr;
-	int toggle = 0, wsize, rc;
+	int toggle = 0, rc;
+	size_t wsize;
 #ifdef _WIN32
 	DWORD len;
 #define DO_WRITE(rc, fd, ptr, w2, len)	rc = WriteFile(fd, ptr, w2, &len, NULL)

From 9e8ee07ddae8f19fe2b78d9a04c85fc4dafed2ff Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 12 Jun 2025 15:16:00 +0100
Subject: [PATCH 458/504] ITS#10355 lmdb mplay: don't assign to stdin/stdout

---
 libraries/liblmdb/mplay.c | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/libraries/liblmdb/mplay.c b/libraries/liblmdb/mplay.c
index 5ca66d9176..c6e981f009 100644
--- a/libraries/liblmdb/mplay.c
+++ b/libraries/liblmdb/mplay.c
@@ -481,12 +481,10 @@ static pidpair *addpid(int tpid)
 	pipe(fdin);
 	if ((pid = fork()) == 0) {
 		/* child */
-		fclose(stdin);
-		fclose(stdout);
+		fflush(stdin);
+		fflush(stdout);
 		dup2(fdout[0], 0);
 		dup2(fdin[1], 1);
-		stdin = fdopen(0, "r");
-		stdout = fdopen(1, "w");
 		child();
 		return 0;	/* NOTREACHED */
 	} else {

From d12c97e2755f2349cca48079dc12718c6dd2c1cb Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 6 Oct 2025 16:49:25 +0100
Subject: [PATCH 459/504] ITS#10396 lmdb: fix mdb_cursor_del0 with multiple
 DUPSORT cursors

Use the correct stack index when adjusting other cursors pointing to
the affected page and the DB has a subDB. Broken in ITS#8406.
---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index b8746bc6c4..887ad3ecfe 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -10325,7 +10325,7 @@ mdb_cursor_del0(MDB_cursor *mc)
 						goto fail;
 				}
 				if (m3->mc_xcursor && !(m3->mc_flags & C_EOF)) {
-					MDB_node *node = NODEPTR(m3->mc_pg[m3->mc_top], m3->mc_ki[m3->mc_top]);
+					MDB_node *node = NODEPTR(m3->mc_pg[mc->mc_top], m3->mc_ki[mc->mc_top]);
 					/* If this node has dupdata, it may need to be reinited
 					 * because its data has moved.
 					 * If the xcursor was not initd it must be reinited.

From b961aca8675907137ba7c5e1fa09185c6cc173cc Mon Sep 17 00:00:00 2001
From: Kris Zyp <kriszyp@gmail.com>
Date: Wed, 26 May 2021 21:43:09 -0600
Subject: [PATCH 460/504] ITS#9564 lmdb: fix race condition freeing spilled
 pages at end of transaction

Move the freeing of the spilled pages list inside the protection of the
transaction mutex so there is no race condition with another transaction
---
 libraries/liblmdb/mdb.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 887ad3ecfe..c37d48ea0b 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -3652,6 +3652,10 @@ mdb_txn_end(MDB_txn *txn, unsigned mode)
 		txn->mt_numdbs = 0;
 		txn->mt_flags = MDB_TXN_FINISHED;
 
+		mdb_midl_free(txn->mt_spill_pgs);
+#if OVERFLOW_NOTYET
+		mdb_mid2l_free(txn->mt_dirty_ovs);
+#endif
 		if (!txn->mt_parent) {
 			mdb_midl_shrink(&txn->mt_free_pgs);
 			env->me_free_pgs = txn->mt_free_pgs;
@@ -3672,10 +3676,6 @@ mdb_txn_end(MDB_txn *txn, unsigned mode)
 			mdb_midl_free(txn->mt_free_pgs);
 			free(txn->mt_u.dirty_list);
 		}
-		mdb_midl_free(txn->mt_spill_pgs);
-#if OVERFLOW_NOTYET
-		mdb_mid2l_free(txn->mt_dirty_ovs);
-#endif
 
 		mdb_midl_free(pghead);
 	}

From 8ba07ad71138d7735df9225638f436dc060266b0 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 16 Dec 2025 17:29:09 +0000
Subject: [PATCH 461/504] ITS#8988 lmdb: make all keys and data 2-byte aligned

This is an incompatible DB format change. Previously, only a single
pad byte was used if key+data was an odd size. Now, separate padding
is used for both odd keys and odd data sizes.

Note: also compile with -DMISALIGNED_OK=0 on x86 to avoid all
misaligned pgno accesses.
---
 libraries/liblmdb/mdb.c | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index c37d48ea0b..2441a03590 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -242,7 +242,7 @@ union semun {
 #define BIG_ENDIAN	__BIG_ENDIAN
 #endif
 
-#if defined(__i386) || defined(__x86_64) || defined(_M_IX86)
+#if (defined(__i386) || defined(__x86_64) || defined(_M_IX86)) && !defined(MISALIGNED_OK)
 #define MISALIGNED_OK	1
 #endif
 
@@ -654,7 +654,7 @@ static txnid_t mdb_debug_start;
 #define MDB_MAGIC	 0xBEEFC0DE
 
 	/**	The version number for a database's datafile format. */
-#define MDB_DATA_VERSION	 ((MDB_DEVEL) ? 999 : 2)
+#define MDB_DATA_VERSION	 ((MDB_DEVEL) ? 999 : 3)
 	/**	The version number for a database's lockfile format. */
 #define MDB_LOCK_VERSION	 ((MDB_DEVEL) ? 999 : 2)
 	/** Number of bits representing #MDB_LOCK_VERSION in #MDB_LOCK_FORMAT.
@@ -1224,7 +1224,7 @@ typedef struct MDB_node {
 	/** Size of a node in a leaf page with a given key and data.
 	 *	This is node header plus key plus data size.
 	 */
-#define LEAFSIZE(k, d)	 (NODESIZE + (k)->mv_size + (d)->mv_size)
+#define LEAFSIZE(k, d)	 (NODESIZE + EVEN((k)->mv_size) + EVEN((d)->mv_size))
 
 	/** Address of node \b i in page \b p */
 #define NODEPTR(p, i)	 ((MDB_node *)((char *)(p) + MP_PTRS(p)[i] + PAGEBASE))
@@ -1233,7 +1233,7 @@ typedef struct MDB_node {
 #define NODEKEY(node)	 (void *)((node)->mn_data)
 
 	/** Address of the data for a node */
-#define NODEDATA(node)	 (void *)((char *)(node)->mn_data + (node)->mn_ksize)
+#define NODEDATA(node)	 (void *)((char *)(node)->mn_data + EVEN((node)->mn_ksize))
 
 	/** Get the page number pointed to by a branch node */
 #define NODEPGNO(node) \
@@ -1253,7 +1253,7 @@ typedef struct MDB_node {
 #define NODEKSZ(node)	 ((node)->mn_ksize)
 
 	/** Copy a page number from src to dst */
-#ifdef MISALIGNED_OK
+#if MISALIGNED_OK
 #define COPY_PGNO(dst,src)	dst = src
 #undef MP_PGNO
 #define MP_PGNO(p)	((p)->mp_pgno)
@@ -9112,7 +9112,7 @@ mdb_node_add(MDB_cursor *mc, indx_t indx,
 
 	room = (ssize_t)SIZELEFT(mp) - (ssize_t)sizeof(indx_t);
 	if (key != NULL)
-		node_size += key->mv_size;
+		node_size += EVEN(key->mv_size);
 	if (IS_LEAF(mp)) {
 		mdb_cassert(mc, key && data);
 		if (F_ISSET(flags, F_BIGDATA)) {
@@ -9133,10 +9133,9 @@ mdb_node_add(MDB_cursor *mc, indx_t indx,
 			flags |= F_BIGDATA;
 			goto update;
 		} else {
-			node_size += data->mv_size;
+			node_size += EVEN(data->mv_size);
 		}
 	}
-	node_size = EVEN(node_size);
 	if ((ssize_t)node_size > room)
 		goto full;
 
@@ -9229,14 +9228,13 @@ mdb_node_del(MDB_cursor *mc, int ksize)
 	}
 
 	node = NODEPTR(mp, indx);
-	sz = NODESIZE + node->mn_ksize;
+	sz = NODESIZE + EVEN(node->mn_ksize);
 	if (IS_LEAF(mp)) {
 		if (F_ISSET(node->mn_flags, F_BIGDATA))
 			sz += sizeof(MDB_ovpage);
 		else
-			sz += NODEDSZ(node);
+			sz += EVEN(NODEDSZ(node));
 	}
-	sz = EVEN(sz);
 
 	ptr = MP_PTRS(mp)[indx];
 	for (i = j = 0; i < numkeys; i++) {

From 7b136c42f1968310ab2124dc45dbd14ef2214dcf Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 6 Jan 2026 20:52:25 +0000
Subject: [PATCH 462/504] ITS#10421 mdb_load: check for malicious input

---
 libraries/liblmdb/mdb_load.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c
index 6d1b5a4328..e6b23406e2 100644
--- a/libraries/liblmdb/mdb_load.c
+++ b/libraries/liblmdb/mdb_load.c
@@ -218,6 +218,12 @@ badend:
 
 	c1 = buf->mv_data;
 	len = strlen((char *)c1);
+	if (!len) {
+		/* This can only happen with an intentionally invalid input
+		 * with a NUL byte after the leading SPACE
+		 */
+		goto badend;
+	}
 	l2 = len;
 
 	/* Is buffer too short? */

From 22206c0f5fb2c86f572d061c490a6736f47772a5 Mon Sep 17 00:00:00 2001
From: Ali Caglayan <alizter@gmail.com>
Date: Sat, 27 Dec 2025 18:26:31 +0000
Subject: [PATCH 463/504] ITS#10419 LMDB: add support for NetBSD

---
 libraries/liblmdb/mdb.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 2441a03590..cffd9cf059 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -179,6 +179,11 @@ typedef SSIZE_T	ssize_t;
 # define MDB_FDATASYNC		fsync
 #endif
 
+/* NetBSD does not define union semun in sys/sem.h */
+#if defined(__NetBSD__) && !defined(_SEM_SEMUN_UNDEFINED)
+# define _SEM_SEMUN_UNDEFINED  1
+#endif
+
 #ifndef _WIN32
 #include <pthread.h>
 #include <signal.h>

From be217bf254aae009a4683204bcd97f97ee4d5ad4 Mon Sep 17 00:00:00 2001
From: Ali Caglayan <alizter@gmail.com>
Date: Sun, 28 Dec 2025 00:34:25 +0000
Subject: [PATCH 464/504] ITS#10420 LMDB: add support for Haiku

---
 libraries/liblmdb/mdb.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index cffd9cf059..626c0c3cf4 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -177,6 +177,9 @@ typedef SSIZE_T	ssize_t;
 # endif
 #elif defined(__ANDROID__)
 # define MDB_FDATASYNC		fsync
+#elif defined(__HAIKU__)
+# define MDB_USE_POSIX_SEM	1
+# define MDB_FDATASYNC		fsync
 #endif
 
 /* NetBSD does not define union semun in sys/sem.h */

From 425525fbec5375220203d94faaf494084b7ee299 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 8 Dec 2025 16:07:00 +0000
Subject: [PATCH 465/504] ITS#9224 lmdb: add support for PREPARE/2-phase commit

---
 libraries/liblmdb/lmdb.h | 55 +++++++++++++++++++++++++++++++++++++-
 libraries/liblmdb/mdb.c  | 57 +++++++++++++++++++++++++++++++++++++---
 2 files changed, 107 insertions(+), 5 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 4ab108116b..aed43a8c66 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -524,8 +524,12 @@ typedef enum MDB_cursor_op {
 #define MDB_CRYPTO_FAIL		(-30777)
 	/** Environment encryption mismatch */
 #define MDB_ENV_ENCRYPTION	(-30776)
+	/** Transaction was already prepared */
+#define MDB_TXN_PENDING		(-30775)
+	/** Environment can't rollback the last transaction */
+#define MDB_CANT_ROLLBACK	(-30774)
 	/** The last defined error code */
-#define MDB_LAST_ERRCODE	MDB_ENV_ENCRYPTION
+#define MDB_LAST_ERRCODE	MDB_CANT_ROLLBACK
 /** @} */
 
 /** @brief Statistics for a database in the environment */
@@ -1128,6 +1132,55 @@ mdb_size_t mdb_txn_id(MDB_txn *txn);
 	 */
 int  mdb_txn_commit(MDB_txn *txn);
 
+	/** @brief Prepare to commit all the operations of a transaction into the database.
+	 *
+	 * This function exists to support two-phase commit protocols.
+	 * All writes in the transaction are persisted to storage, but the final
+	 * metapage update is not performed. All cursors on the transaction will be
+	 * closed. Only #mdb_txn_abort() or #mdb_txn_commit() are valid after this
+	 * call. It is assumed that once the regular data pages are successfully written
+	 * by this call, the metapage update from a subsequent commit cannot fail, but
+	 * hardware-level media failures could still break this assumption.
+	 * @param[in] txn A transaction handle returned by #mdb_txn_begin()
+	 * @return A non-zero error value on failure and 0 on success. Some possible
+	 * errors are:
+	 * <ul>
+	 *	<li>EINVAL - an invalid parameter was specified.
+	 *	<li>ENOSPC - no more disk space.
+	 *	<li>EIO - a low-level I/O error occurred while writing.
+	 *	<li>ENOMEM - out of memory.
+	 *	<li>#MDB_TXN_PENDING - the transaction has already been prepared.
+	 *		It can only be aborted or committed.
+	 * </ul>
+	 */
+int  mdb_txn_prepare(MDB_txn *txn);
+
+	/** @brief Rollback the last committed transaction in the environment.
+	 *
+	 * This function exists to support two-phase commit protocols.
+	 * The metapage update for the last committed transaction will be zeroed,
+	 * so its changes will be ignored. It should only be used when the local
+	 * phase of a multi-phase transaction has fully committed, but some other
+	 * remote phase which successfully prepared has failed to commit.
+	 * This function may not be called twice in a row. No other operations
+	 * may be performed on the environment, by any processes, between the
+	 * preceding #mdb_txn_commit() and this call.
+	 * @param[in] env An environment handle returned by #mdb_env_create().
+	 * @param[in] txnid The ID of the transaction to rollback, obtained from
+	 * #mdb_txnid() on the previous transaction.
+	 * @return A non-zero error value on failure and 0 on success. Some possible
+	 * errors are:
+	 * <ul>
+	 *	<li>EINVAL - an invalid parameter was specified.
+	 *	<li>ENOSPC - no more disk space.
+	 *	<li>EIO - a low-level I/O error occurred while writing.
+	 *	<li>#MDB_CANT_ROLLBACK - a rollback has already been done, there is
+	 *		no other valid metapage to roll back to, or another transaction
+	 *		has already been committed over the specified txnid.
+	 * </ul>
+	 */
+int  mdb_env_rollback(MDB_env *env, mdb_size_t txnid);
+
 	/** @brief Abandon all the operations of the transaction instead of saving them.
 	 *
 	 * The transaction handle is freed. It and its cursors must not be used
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 626c0c3cf4..bfecbb5b9a 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1500,8 +1500,9 @@ struct MDB_txn {
 #define MDB_TXN_SPILLS		0x08		/**< txn or a parent has spilled pages */
 #define MDB_TXN_HAS_CHILD	0x10		/**< txn has an #MDB_txn.%mt_child */
 #define MDB_TXN_DIRTYNUM	0x20		/**< dirty list uses nump list */
+#define MDB_TXN_PREPARE		0x40		/**< prepare txn, don't fully commit */
 	/** most operations on the txn are currently illegal */
-#define MDB_TXN_BLOCKED		(MDB_TXN_FINISHED|MDB_TXN_ERROR|MDB_TXN_HAS_CHILD)
+#define MDB_TXN_BLOCKED		(MDB_TXN_FINISHED|MDB_TXN_ERROR|MDB_TXN_HAS_CHILD|MDB_TXN_PREPARE)
 /** @} */
 	unsigned int	mt_flags;		/**< @ref mdb_txn */
 	/** #dirty_list room: Array size - \#dirty pages visible to this txn.
@@ -1888,6 +1889,8 @@ static char *const mdb_errstr[] = {
 	"MDB_BAD_CHECKSUM: Page checksum mismatch",
 	"MDB_CRYPTO_FAIL: Page encryption or decryption failed",
 	"MDB_ENV_ENCRYPTION: Environment encryption mismatch",
+	"MDB_TXN_PENDING: Transaction already prepared, must abort or commit",
+	"MDB_CANT_ROLLBACK: Environment can't rollback last transaction",
 };
 
 char *
@@ -4263,7 +4266,7 @@ done:
 static int ESECT mdb_env_share_locks(MDB_env *env, int *excl);
 
 static int
-_mdb_txn_commit(MDB_txn *txn)
+_mdb_txn_commit(MDB_txn *txn, int flag)
 {
 	int		rc;
 	unsigned int i, end_mode;
@@ -4276,7 +4279,7 @@ _mdb_txn_commit(MDB_txn *txn)
 	end_mode = MDB_END_EMPTY_COMMIT|MDB_END_UPDATE|MDB_END_SLOT|MDB_END_FREE;
 
 	if (txn->mt_child) {
-		rc = _mdb_txn_commit(txn->mt_child);
+		rc = _mdb_txn_commit(txn->mt_child, 0);
 		if (rc)
 			goto fail;
 	}
@@ -4287,6 +4290,10 @@ _mdb_txn_commit(MDB_txn *txn)
 		goto done;
 	}
 
+	if (F_ISSET(txn->mt_flags, MDB_TXN_PREPARE)) {
+		goto prepared;
+	}
+
 	if (txn->mt_flags & (MDB_TXN_FINISHED|MDB_TXN_ERROR)) {
 		DPUTS("txn has failed/finished, can't commit");
 		if (txn->mt_parent)
@@ -4486,6 +4493,12 @@ _mdb_txn_commit(MDB_txn *txn)
 	if (!F_ISSET(txn->mt_flags, MDB_TXN_NOSYNC) &&
 		(rc = mdb_env_sync0(env, 0, txn->mt_next_pgno)))
 		goto fail;
+	if (F_ISSET(flag, MDB_TXN_PREPARE)) {
+		txn->mt_flags |= MDB_TXN_PREPARE;
+		return MDB_SUCCESS;
+	}
+
+prepared:
 	if ((rc = mdb_env_write_meta(txn)))
 		goto fail;
 	end_mode = MDB_END_COMMITTED|MDB_END_UPDATE;
@@ -4512,7 +4525,43 @@ int
 mdb_txn_commit(MDB_txn *txn)
 {
 	MDB_TRACE(("%p", txn));
-	return _mdb_txn_commit(txn);
+	return _mdb_txn_commit(txn, 0);
+}
+
+int
+mdb_txn_prepare(MDB_txn *txn)
+{
+	MDB_TRACE(("%p", txn));
+	if (F_ISSET(txn->mt_flags, MDB_TXN_PREPARE))
+		return MDB_TXN_PENDING;
+	return _mdb_txn_commit(txn, MDB_TXN_PREPARE);
+}
+
+int
+mdb_env_rollback(MDB_env *env, mdb_size_t txnid)
+{
+	MDB_meta **metas = env->me_metas;
+	int newest, previous, rc = 0;
+
+	if (env->me_txns && LOCK_MUTEX(rc, env, env->me_wmutex))
+		return rc;
+	newest = metas[0]->mm_txnid < metas[1]->mm_txnid;
+	previous = newest ^ 1;
+	if (!metas[previous]->mm_txnid || metas[newest]->mm_txnid != txnid)
+		rc = MDB_CANT_ROLLBACK;
+	else {
+		MDB_txn txn = {0};
+		MDB_db dbs[2] = {0};
+		txn.mt_env = env;
+		txn.mt_dbs = dbs;
+		rc = mdb_env_write_meta(&txn);
+	}
+	if (env->me_txns) {
+		if (rc == MDB_SUCCESS)
+			env->me_txns->mti_txnid = metas[previous]->mm_txnid;
+		UNLOCK_MUTEX(env->me_wmutex);
+	}
+	return rc;
 }
 
 static int ESECT mdb_env_map(MDB_env *env, void *addr);

From 10161b20fef84de5a07afdcbce3b59aea1123a6e Mon Sep 17 00:00:00 2001
From: Kerollmops <clement@meilisearch.com>
Date: Mon, 30 Mar 2026 17:48:04 +0200
Subject: [PATCH 466/504] ITS#10395 LMDB: Allow multiple nested read txns from
 a write txn

---
 libraries/liblmdb/lmdb.h |  10 ++--
 libraries/liblmdb/mdb.c  | 112 +++++++++++++++++++++++++++++----------
 2 files changed, 91 insertions(+), 31 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index aed43a8c66..dd77ca9c60 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -1070,6 +1070,10 @@ int mdb_env_set_checksum(MDB_env *env, MDB_sum_func *func, unsigned int size);
 	 * as its parent. Transactions may be nested to any level. A parent
 	 * transaction and its cursors may not issue any other operations than
 	 * mdb_txn_commit and mdb_txn_abort while it has active child transactions.
+	 * @note A parent transaction with read-only child transactions must not
+	 * issue any operations while it has active child transactions, not even
+	 * #mdb_txn_commit() nor #mdb_txn_abort(). Child transactions must be
+	 * aborted prior to performing actions with the parent one.
 	 * @param[in] flags Special options for this transaction. This parameter
 	 * must be set to 0 or by bitwise OR'ing together one or more of the
 	 * values described here.
@@ -1191,7 +1195,7 @@ int  mdb_env_rollback(MDB_env *env, mdb_size_t txnid);
 	 */
 void mdb_txn_abort(MDB_txn *txn);
 
-	/** @brief Reset a read-only transaction.
+	/** @brief Reset a non-nested read-only transaction.
 	 *
 	 * Abort the transaction like #mdb_txn_abort(), but keep the transaction
 	 * handle. #mdb_txn_renew() may reuse the handle. This saves allocation
@@ -1210,11 +1214,11 @@ void mdb_txn_abort(MDB_txn *txn);
 	 */
 void mdb_txn_reset(MDB_txn *txn);
 
-	/** @brief Renew a read-only transaction.
+	/** @brief Renew a non-nested read-only transaction.
 	 *
 	 * This acquires a new reader lock for a transaction handle that had been
 	 * released by #mdb_txn_reset(). It must be called before a reset transaction
-	 * may be used again.
+	 * may be used again. Nested read-only transactions cannot be renewed.
 	 * @param[in] txn A transaction handle returned by #mdb_txn_begin()
 	 * @return A non-zero error value on failure and 0 on success. Some possible
 	 * errors are:
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index bfecbb5b9a..8ea3213773 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -374,6 +374,8 @@ typedef HANDLE mdb_mutex_t, mdb_mutexref_t;
 #define pthread_setspecific(x,y)	(TlsSetValue(x,y) ? 0 : ErrCode())
 #define pthread_mutex_unlock(x)	ReleaseMutex(*x)
 #define pthread_mutex_lock(x)	WaitForSingleObject(*x, INFINITE)
+#define pthread_mutex_init(x,y)	((*x = CreateMutex(NULL, FALSE, NULL)) ? 0 : ErrCode())
+#define pthread_mutex_destroy(x)	(CloseHandle(*x) ? 0 : ErrCode())
 #define pthread_cond_signal(x)	SetEvent(*x)
 #define pthread_cond_wait(cond,mutex)	do{SignalObjectAndWait(*mutex, *cond, INFINITE, FALSE); WaitForSingleObject(*mutex, INFINITE);}while(0)
 #define THREAD_CREATE(thr,start,arg) \
@@ -1391,6 +1393,10 @@ struct MDB_txn {
 	MDB_txn		*mt_parent;		/**< parent of a nested txn */
 	/** Nested txn under this txn, set together with flag #MDB_TXN_HAS_CHILD */
 	MDB_txn		*mt_child;
+	/** The count of nested RDONLY txns under this txn */
+	unsigned int	mt_rdonly_child_count;
+	/** Mutex protecting mt_rdonly_child_count */
+	pthread_mutex_t	mt_child_mutex;
 	pgno_t		mt_next_pgno;	/**< next unallocated page */
 #if MDB_RPAGE_CACHE
 	pgno_t		mt_last_pgno;	/**< last written page */
@@ -3347,6 +3353,8 @@ mdb_txn_renew0(MDB_txn *txn)
 			mdb_debug = MDB_DBG_INFO;
 #endif
 		txn->mt_child = NULL;
+		txn->mt_rdonly_child_count = 0;
+		pthread_mutex_init(&txn->mt_child_mutex, NULL);
 		txn->mt_loose_pgs = NULL;
 		txn->mt_loose_count = 0;
 		if (env->me_flags & MDB_WRITEMAP) {
@@ -3461,11 +3469,22 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 		return EACCES;
 
 	if (parent) {
-		/* Nested transactions: Max 1 child, write txns only, no writemap */
+		/* Nested transactions:
+		 * Only write txns may have nested txns;
+		 * if the nested txn is a write txn there may only be 1, no writemap;
+		 * if the nested txn is a read txn there may be arbitrarily many.
+		 */
+		pthread_mutex_lock(&parent->mt_child_mutex);
 		flags |= parent->mt_flags;
-		if (flags & (MDB_RDONLY|MDB_WRITEMAP|MDB_TXN_BLOCKED)) {
-			return (parent->mt_flags & MDB_TXN_RDONLY) ? EINVAL : MDB_BAD_TXN;
+		if (parent->mt_child && F_ISSET(parent->mt_child->mt_flags, MDB_RDONLY) && F_ISSET(flags, MDB_RDONLY)) {
+			flags &= ~MDB_TXN_HAS_CHILD;
 		}
+		if ((F_ISSET(flags, MDB_WRITEMAP) && !F_ISSET(flags, MDB_RDONLY)) || F_ISSET(flags, MDB_TXN_BLOCKED)) {
+			flags = parent->mt_flags;
+			pthread_mutex_unlock(&parent->mt_child_mutex);
+			return (flags & MDB_TXN_RDONLY) ? EINVAL : MDB_BAD_TXN;
+		}
+		pthread_mutex_unlock(&parent->mt_child_mutex);
 		/* Child txns save MDB_pgstate and use own copy of cursors */
 		size = env->me_maxdbs * (sizeof(MDB_db)+sizeof(MDB_cursor *)+1);
 		size += tsize = sizeof(MDB_ntxn);
@@ -3508,24 +3527,35 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 		txn->mt_workid = txn->mt_last_workid = workid;
 		txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs);
 		txn->mt_dbiseqs = parent->mt_dbiseqs;
-		txn->mt_u.dirty_list = malloc(sizeof(MDB_ID2)*MDB_IDL_UM_SIZE);
-		if (!txn->mt_u.dirty_list ||
-			!(txn->mt_free_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX)))
-		{
-			free(txn->mt_u.dirty_list);
-			free(txn);
-			return ENOMEM;
+		/* share parent dirty and free pages with nested RDONLY txn */
+		if (F_ISSET(flags, MDB_RDONLY)) {
+			txn->mt_u.dirty_list = parent->mt_u.dirty_list;
+			txn->mt_free_pgs = parent->mt_free_pgs;
+		} else {
+			txn->mt_u.dirty_list = malloc(sizeof(MDB_ID2)*MDB_IDL_UM_SIZE);
+			if (!txn->mt_u.dirty_list ||
+				!(txn->mt_free_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX)))
+			{
+				free(txn->mt_u.dirty_list);
+				free(txn);
+				return ENOMEM;
+			}
+			txn->mt_u.dirty_list[0].mid = 0;
 		}
 		txn->mt_txnid = parent->mt_txnid;
 		txn->mt_dirty_room = parent->mt_dirty_room;
-		txn->mt_u.dirty_list[0].mid = 0;
 		txn->mt_spill_pgs = NULL;
 #if OVERFLOW_NOTYET
 		txn->mt_dirty_ovs = NULL;
 #endif
 		txn->mt_next_pgno = parent->mt_next_pgno;
+		pthread_mutex_lock(&parent->mt_child_mutex);
 		parent->mt_flags |= MDB_TXN_HAS_CHILD;
 		parent->mt_child = txn;
+		if (F_ISSET(flags, MDB_RDONLY)) {
+			parent->mt_rdonly_child_count++;
+		}
+		pthread_mutex_unlock(&parent->mt_child_mutex);
 		txn->mt_parent = parent;
 		txn->mt_numdbs = parent->mt_numdbs;
 #if MDB_RPAGE_CACHE
@@ -3538,19 +3568,22 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 			txn->mt_dbflags[i] = parent->mt_dbflags[i] & ~DB_NEW;
 		rc = 0;
 		ntxn = (MDB_ntxn *)txn;
-		ntxn->mnt_pgstate = env->me_pgstate; /* save parent me_pghead & co */
-		if (env->me_pghead) {
-			size = MDB_IDL_SIZEOF(env->me_pghead);
-			env->me_pghead = mdb_midl_alloc(env->me_pghead[0]);
-			if (env->me_pghead)
-				memcpy(env->me_pghead, ntxn->mnt_pgstate.mf_pghead, size);
-			else
-				rc = ENOMEM;
+		if (!F_ISSET(flags, MDB_RDONLY)) {
+			ntxn->mnt_pgstate = env->me_pgstate; /* save parent me_pghead & co */
+			/* Do not copy parent me_pghead when nested and RDONLY */
+			if (env->me_pghead) {
+				size = MDB_IDL_SIZEOF(env->me_pghead);
+				env->me_pghead = mdb_midl_alloc(env->me_pghead[0]);
+				if (env->me_pghead)
+					memcpy(env->me_pghead, ntxn->mnt_pgstate.mf_pghead, size);
+				else
+					rc = ENOMEM;
+			}
+			if (!rc)
+				rc = mdb_cursor_shadow(parent, txn);
+			if (rc)
+				mdb_txn_end(txn, MDB_END_FAIL_BEGINCHILD);
 		}
-		if (!rc)
-			rc = mdb_cursor_shadow(parent, txn);
-		if (rc)
-			mdb_txn_end(txn, MDB_END_FAIL_BEGINCHILD);
 	} else { /* MDB_RDONLY */
 		txn->mt_dbiseqs = env->me_dbiseqs;
 renew:
@@ -3559,7 +3592,9 @@ renew:
 	if (rc) {
 		if (txn != env->me_txn0) {
 			/* mt_rpages belongs to parent */
-			free(txn->mt_u.dirty_list);
+			if (!F_ISSET(flags, MDB_RDONLY)) {
+				free(txn->mt_u.dirty_list);
+			}
 			free(txn);
 		}
 	} else {
@@ -3639,6 +3674,16 @@ mdb_txn_end(MDB_txn *txn, unsigned mode)
 		(void *) txn, (void *)env, txn->mt_dbs[MAIN_DBI].md_root));
 
 	if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) {
+		if (txn->mt_parent) {
+				pthread_mutex_lock(&txn->mt_parent->mt_child_mutex);
+				txn->mt_parent->mt_rdonly_child_count--;
+				/* mark parent txn has no longer having children if this is the last nested txn */
+				if (txn->mt_parent->mt_rdonly_child_count == 0) {
+					txn->mt_parent->mt_child = NULL;
+					txn->mt_parent->mt_flags &= ~MDB_TXN_HAS_CHILD;
+				}
+				pthread_mutex_unlock(&txn->mt_parent->mt_child_mutex);
+		} else
 		if (txn->mt_u.reader) {
 			txn->mt_u.reader->mr_txnid = (txnid_t)-1;
 			if (!(env->me_flags & MDB_NOTLS)) {
@@ -3725,8 +3770,11 @@ mdb_txn_end(MDB_txn *txn, unsigned mode)
 			free(tl);
 	}
 #endif
-	if (mode & MDB_END_FREE)
+	if (mode & MDB_END_FREE) {
+		if (!F_ISSET(txn->mt_flags, MDB_TXN_RDONLY))
+			pthread_mutex_destroy(&txn->mt_child_mutex);
 		free(txn);
+	}
 }
 
 void
@@ -3735,8 +3783,8 @@ mdb_txn_reset(MDB_txn *txn)
 	if (txn == NULL)
 		return;
 
-	/* This call is only valid for read-only txns */
-	if (!(txn->mt_flags & MDB_TXN_RDONLY))
+	/* This call is only valid for non-nested read-only txns */
+	if (!(txn->mt_flags & MDB_TXN_RDONLY) || txn->mt_parent)
 		return;
 
 	mdb_txn_end(txn, MDB_END_RESET);
@@ -3748,6 +3796,14 @@ _mdb_txn_abort(MDB_txn *txn)
 	if (txn == NULL)
 		return;
 
+	if (txn->mt_parent && (txn->mt_flags & MDB_RDONLY)) {
+		/* You must first abort the child before the parent */
+		pthread_mutex_lock(&txn->mt_child_mutex);
+		int count = txn->mt_rdonly_child_count;
+		pthread_mutex_unlock(&txn->mt_child_mutex);
+		mdb_tassert(txn, txn->mt_parent && count == 0);
+	}
+
 	if (txn->mt_child)
 		_mdb_txn_abort(txn->mt_child);
 
@@ -7216,7 +7272,7 @@ mdb_page_get(MDB_cursor *mc, pgno_t pgno,
 	MDB_txn *txn = mc->mc_txn, *tx2;
 	MDB_page *p = NULL;
 
-	if (! (mc->mc_flags & (C_ORIG_RDONLY|C_WRITEMAP))) {
+	if (! (( mc->mc_flags & (C_ORIG_RDONLY|C_WRITEMAP) ) && mc->mc_txn->mt_parent == NULL)) {
 		for (tx2 = txn;; ) {
 			MDB_ID2L dl = tx2->mt_u.dirty_list;
 			MDB_IDL  sl;

From b0facd028859ff463122c53e02584dbf60ba9a50 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 15 Apr 2026 15:14:37 +0100
Subject: [PATCH 467/504] ITS#10054 lmdb: break up large overflow page writes

We were ignoring MAX_WRITE when a single overflow page was larger.
Now we guarantee that all writes are <= MAX_WRITE (1GB).
---
 libraries/liblmdb/mdb.c | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 8ea3213773..c785f13042 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -4198,6 +4198,14 @@ retry_write:
 				wres = pwritev(fd, iov, n, wpos);
 #else
 				if (n == 1) {
+					while (wsize > MAX_WRITE) {
+						wsize -= MAX_WRITE;
+						wres = pwrite(fd, iov[0].iov_base, MAX_WRITE, wpos);
+						if (wres != MAX_WRITE)
+							goto bad_write;;
+						wpos += MAX_WRITE;
+						iov[0].iov_base += MAX_WRITE;
+					}
 					wres = pwrite(fd, iov[0].iov_base, wsize, wpos);
 				} else {
 retry_seek:
@@ -4213,6 +4221,7 @@ retry_seek:
 					}
 				}
 #endif
+bad_write:
 				if (wres != wsize) {
 					if (wres < 0) {
 						rc = ErrCode();

From 7025c4ea3458408a60e8d4e9ec0ee89327d6e146 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 15 Apr 2026 15:24:01 +0100
Subject: [PATCH 468/504] ITS#10054 lmdb: more for prev commit

---
 libraries/liblmdb/mdb.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index c785f13042..6247f867ab 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -4194,9 +4194,6 @@ retry_write:
 				}
 				async_i++;
 #else /* _WIN32 */
-#ifdef MDB_USE_PWRITEV
-				wres = pwritev(fd, iov, n, wpos);
-#else
 				if (n == 1) {
 					while (wsize > MAX_WRITE) {
 						wsize -= MAX_WRITE;
@@ -4208,6 +4205,9 @@ retry_write:
 					}
 					wres = pwrite(fd, iov[0].iov_base, wsize, wpos);
 				} else {
+#ifdef MDB_USE_PWRITEV
+					wres = pwritev(fd, iov, n, wpos);
+#else
 retry_seek:
 					if (lseek(fd, wpos, SEEK_SET) == -1) {
 						rc = ErrCode();

From 06680017d3560ccd24e7abd3881510c2b46c3b7f Mon Sep 17 00:00:00 2001
From: Philipp Storz <philipp.storz@bareos.com>
Date: Mon, 14 Mar 2016 10:07:35 +0100
Subject: [PATCH 469/504] ITS#8386 lmdb: be a bit more precise that mdb_get
 retrieves data in intro.doc

---
 libraries/liblmdb/intro.doc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/intro.doc b/libraries/liblmdb/intro.doc
index b5bb06716a..99081bda7b 100644
--- a/libraries/liblmdb/intro.doc
+++ b/libraries/liblmdb/intro.doc
@@ -51,7 +51,7 @@ databases should only be opened once, by the first transaction in
 the process. After the first transaction completes, the database
 handles can freely be used by all subsequent transactions.
 
-Within a transaction, #mdb_get() and #mdb_put() can store single
+Within a transaction, #mdb_get() can retrieve and #mdb_put() can store single
 key/value pairs if that is all you need to do (but see \ref Cursors
 below if you want to do more).
 

From 5b3df47632805de52e772bb14bcdece04219bfe9 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 15 Apr 2026 18:17:55 +0100
Subject: [PATCH 470/504] ITS#8824 mdb_dump: cleanup check for MDB_SUCCESS

---
 libraries/liblmdb/mdb_dump.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb_dump.c b/libraries/liblmdb/mdb_dump.c
index 8ccd04833d..267334b069 100644
--- a/libraries/liblmdb/mdb_dump.c
+++ b/libraries/liblmdb/mdb_dump.c
@@ -131,7 +131,7 @@ static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name)
 	rc = mdb_cursor_open(txn, dbi, &mc);
 	if (rc) return rc;
 
-	while ((rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT) == MDB_SUCCESS)) {
+	while ((rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT)) == MDB_SUCCESS) {
 		if (gotsig) {
 			rc = EINTR;
 			break;

From e30a2d1672740632a576d023b99142912127e541 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 15 Apr 2026 19:15:43 +0100
Subject: [PATCH 471/504] ITS#10108 mdb_dump: fix comment about -a option

No change in behavior, manpage doc was always correct.
---
 libraries/liblmdb/mdb_dump.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb_dump.c b/libraries/liblmdb/mdb_dump.c
index 267334b069..fac85d0f8c 100644
--- a/libraries/liblmdb/mdb_dump.c
+++ b/libraries/liblmdb/mdb_dump.c
@@ -174,7 +174,7 @@ int main(int argc, char *argv[])
 		usage(prog);
 	}
 
-	/* -a: dump main DB and all subDBs
+	/* -a: dump all subDBs
 	 * -s: dump only the named subDB
 	 * -n: use NOSUBDIR flag on env_open
 	 * -p: use printable characters

From bd9cd548b02e77fea8e9ba5d03993b1a688ef223 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 15 Apr 2026 19:39:36 +0100
Subject: [PATCH 472/504] ITS#10454 lmdb: can't rely on O_DSYNC on MacOS

So we must explicitly sync writes to the metapage.
---
 libraries/liblmdb/mdb.c | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 6247f867ab..1c885e8dd6 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -4865,6 +4865,12 @@ mdb_env_write_meta(MDB_txn *txn)
 				rc = ErrCode();
 				goto fail;
 			}
+#if defined(__APPLE__)
+			if (MDB_FDATASYNC(env->me_mfd)) {
+				rc = ErrCode();
+				goto fail;
+			}
+#endif
 		}
 		goto done;
 	}
@@ -4924,6 +4930,12 @@ fail:
 		env->me_flags |= MDB_FATAL_ERROR;
 		return rc;
 	}
+#if defined(__APPLE__)
+	if (mfd == env->me_mfd && MDB_FDATASYNC(env->me_mfd)) {
+		rc = ErrCode();
+		return rc;
+	}
+#endif
 	/* MIPS has cache coherency issues, this is a no-op everywhere else */
 	CACHEFLUSH(env->me_map + off, len, DCACHE);
 done:

From b2cfdd23a2764ecd92b5c9fdb4950a48f7825a3e Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 15 Apr 2026 20:03:06 +0100
Subject: [PATCH 473/504] ITS#8739 lmdb: No fdatasync on FreeBSD 11.0 and older

---
 libraries/liblmdb/mdb.c | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 1c885e8dd6..1e84c8383e 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -160,9 +160,14 @@ typedef SSIZE_T	ssize_t;
 #include <resolv.h>	/* defines BYTE_ORDER on HPUX and Solaris */
 #endif
 
-#if defined(__FreeBSD__) && defined(__FreeBSD_version) && __FreeBSD_version >= 1100110
-# define MDB_USE_POSIX_MUTEX	1
-# define MDB_USE_ROBUST	1
+#if defined(__FreeBSD__) && defined(__FreeBSD_version)
+# if __FreeBSD_version >= 1100110
+#  define MDB_USE_POSIX_MUTEX	1
+#  define MDB_USE_ROBUST	1
+# endif
+# if __FreeBSD_version < 1101000
+#  define MDB_FDATASYNC		fsync
+# endif
 #elif defined(__APPLE__) || defined (BSD) || defined(__FreeBSD_kernel__)
 # if defined(__APPLE__) && !defined(MDB_USE_ROBUST)
 # define MDB_USE_POSIX_SEM	1

From 0da8bdd9409b42d5279547ca53969735109c6883 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 21 Apr 2026 16:39:49 +0100
Subject: [PATCH 474/504] ITS#10275 mdb_load: add -Q to usage message

---
 libraries/liblmdb/mdb_load.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c
index e6b23406e2..b0f06b8f37 100644
--- a/libraries/liblmdb/mdb_load.c
+++ b/libraries/liblmdb/mdb_load.c
@@ -295,7 +295,7 @@ badend:
 
 static void usage(void)
 {
-	fprintf(stderr, "usage: %s [-V] [-a] [-f input] [-n] [-m module [-w password]] [-s name] [-N] [-T] dbpath\n", prog);
+	fprintf(stderr, "usage: %s [-V] [-a] [-f input] [-n] [-m module [-w password]] [-s name] [-N] [-Q] [-T] dbpath\n", prog);
 	exit(EXIT_FAILURE);
 }
 

From 4fadb7075c6f8510947d95985a061cbb0162877d Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 21 Apr 2026 16:43:09 +0100
Subject: [PATCH 475/504] ITS#9223 LMDB: add incremental dump/load

---
 libraries/liblmdb/lmdb.h     |  49 +++++++
 libraries/liblmdb/mdb.c      | 239 ++++++++++++++++++++++++++++++-----
 libraries/liblmdb/mdb_dump.1 |   8 ++
 libraries/liblmdb/mdb_dump.c |  38 +++++-
 libraries/liblmdb/mdb_load.1 |   5 +
 libraries/liblmdb/mdb_load.c |  32 ++++-
 6 files changed, 330 insertions(+), 41 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index dd77ca9c60..6376a6b555 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -791,6 +791,55 @@ int  mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags);
 	 */
 int  mdb_env_copyfd2(MDB_env *env, mdb_filehandle_t fd, unsigned int flags);
 
+	/** @brief Perform incremental dump of an LMDB environment to the
+	 *	specified file descriptor.
+	 *
+	 * This function may be used to make an incremental backup of an
+	 * existing environment.
+	 * @note This call can trigger significant file size growth if run in
+	 * parallel with write transactions, because it employs a read-only
+	 * transaction. See long-lived transactions under @ref caveats_sec.
+	 * @param[in] env An environment handle returned by #mdb_env_create(). It
+	 * must have already been opened successfully.
+	 * @param[in] fd The filedescriptor to write the copy to. It must
+	 * have already been opened for Write access.
+	 * @param[in] txnid The transaction ID of a previous backup. It must
+	 * be greater than zero.
+	 * @return A non-zero error value on failure and 0 on success.
+	 */
+int  mdb_env_incr_dumpfd(MDB_env *env, mdb_filehandle_t fd, size_t txnid);
+
+	/** @brief Perform incremental dump of an LMDB environment to the
+	 *	specified file.
+	 *
+	 * This function may be used to make an incremental backup of an
+	 * existing environment.
+	 * @note This call can trigger significant file size growth if run in
+	 * parallel with write transactions, because it employs a read-only
+	 * transaction. See long-lived transactions under @ref caveats_sec.
+	 * @param[in] env An environment handle returned by #mdb_env_create(). It
+	 * must have already been opened successfully.
+	 * @param[in] path The name of the file to write the copy to. It must
+	 * not already exist.
+	 * @param[in] txnid The transaction ID of a previous backup. It must
+	 * be greater than zero.
+	 * @return A non-zero error value on failure and 0 on success.
+	 */
+int  mdb_env_incr_dump(MDB_env *env, const char *path, size_t txnid);
+
+	/** @brief Reload an incremental dump of an LMDB environment from the
+	 *	specified file descriptor.
+	 *
+	 * This function may be used to load an incremental backup of an
+	 * existing environment.
+	 * @note No other tasks may access the environment while this runs.
+	 * @param[in] env An environment handle returned by #mdb_env_create(). It
+	 * must have already been opened successfully, using #MDB_WRITEMAP.
+	 * @param[in] fd The filedescriptor to read the backup from.
+	 * @return A non-zero error value on failure and 0 on success.
+	 */
+int  mdb_env_incr_loadfd(MDB_env *env, mdb_filehandle_t fd);
+
 	/** @brief Return statistics about the LMDB environment.
 	 *
 	 * @param[in] env An environment handle returned by #mdb_env_create()
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 1e84c8383e..82e5863790 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -155,6 +155,18 @@ typedef SSIZE_T	ssize_t;
 #endif
 #endif
 
+#ifdef _WIN32
+#define ALIGNED_FREE(x)	_aligned_free(x)
+#define MEMALIGN(rc, buf, align, size) do { buf=_aligned_malloc(size, align); rc = (buf == NULL) ? ERROR_NOT_ENOUGH_MEMORY : 0; } while(0)
+#else
+#define ALIGNED_FREE(x)	free(x)
+# ifdef HAVE_MEMALIGN
+# define MEMALIGN(rc, buf, align, size) do { buf=memalign(align, size); rc = (buf == NULL) ? errno : 0; } while(0)
+# else
+# define MEMALIGN(rc, buf, align, size) rc = posix_memalign((void **)&buf, align, size)
+# endif
+#endif
+
 #if !(defined(BYTE_ORDER) || defined(__BYTE_ORDER))
 #include <netinet/in.h>
 #include <resolv.h>	/* defines BYTE_ORDER on HPUX and Solaris */
@@ -11044,6 +11056,12 @@ typedef struct mdb_copy {
 	volatile int mc_error;
 } mdb_copy;
 
+#ifdef _WIN32
+#define DO_WRITE(rc, fd, ptr, w2, len)	rc = WriteFile(fd, ptr, w2, &len, NULL)
+#else
+#define DO_WRITE(rc, fd, ptr, w2, len)	len = write(fd, ptr, w2); rc = (len >= 0)
+#endif
+
 	/** Dedicated writer thread for compacting copy. */
 static THREAD_RET ESECT CALL_CONV
 mdb_env_copythr(void *arg)
@@ -11054,10 +11072,8 @@ mdb_env_copythr(void *arg)
 	size_t wsize;
 #ifdef _WIN32
 	DWORD len;
-#define DO_WRITE(rc, fd, ptr, w2, len)	rc = WriteFile(fd, ptr, w2, &len, NULL)
 #else
 	int len;
-#define DO_WRITE(rc, fd, ptr, w2, len)	len = write(fd, ptr, w2); rc = (len >= 0)
 #ifdef SIGPIPE
 	sigset_t set;
 	sigemptyset(&set);
@@ -11119,7 +11135,6 @@ again:
 	}
 	pthread_mutex_unlock(&my->mc_mutex);
 	return (THREAD_RET)0;
-#undef DO_WRITE
 }
 
 	/** Give buffer and/or #MDB_EOF to writer thread, await unused buffer.
@@ -11324,32 +11339,16 @@ mdb_env_copyfd1(MDB_env *env, HANDLE fd)
 		rc = ErrCode();
 		goto done;
 	}
-	my.mc_wbuf[0] = _aligned_malloc(MDB_WBUF*2, env->me_os_psize);
-	if (my.mc_wbuf[0] == NULL) {
-		/* _aligned_malloc() sets errno, but we use Windows error codes */
-		rc = ERROR_NOT_ENOUGH_MEMORY;
-		goto done;
-	}
 #else
 	if ((rc = pthread_mutex_init(&my.mc_mutex, NULL)) != 0)
 		return rc;
 	if ((rc = pthread_cond_init(&my.mc_cond, NULL)) != 0)
 		goto done2;
-#ifdef HAVE_MEMALIGN
-	my.mc_wbuf[0] = memalign(env->me_os_psize, MDB_WBUF*2);
-	if (my.mc_wbuf[0] == NULL) {
-		rc = errno;
+#endif
+	MEMALIGN(rc, my.mc_wbuf[0], env->me_os_psize, MDB_WBUF*2);
+	if (rc)
 		goto done;
-	}
-#else
-	{
-		void *p;
-		if ((rc = posix_memalign(&p, env->me_os_psize, MDB_WBUF*2)) != 0)
-			goto done;
-		my.mc_wbuf[0] = p;
-	}
-#endif
-#endif
+
 	memset(my.mc_wbuf[0], 0, MDB_WBUF*2);
 	my.mc_wbuf[1] = my.mc_wbuf[0] + MDB_WBUF;
 	my.mc_next_pgno = NUM_METAS;
@@ -11425,15 +11424,14 @@ finish:
 
 done:
 #ifdef _WIN32
-	if (my.mc_wbuf[0]) _aligned_free(my.mc_wbuf[0]);
 	if (my.mc_cond)  CloseHandle(my.mc_cond);
 	if (my.mc_mutex) CloseHandle(my.mc_mutex);
 #else
-	free(my.mc_wbuf[0]);
 	pthread_cond_destroy(&my.mc_cond);
 done2:
 	pthread_mutex_destroy(&my.mc_mutex);
 #endif
+	if (my.mc_wbuf[0]) ALIGNED_FREE(my.mc_wbuf[0]);
 	return rc ? rc : my.mc_error;
 }
 
@@ -11448,11 +11446,9 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd)
 	char *ptr;
 #ifdef _WIN32
 	DWORD len, w2;
-#define DO_WRITE(rc, fd, ptr, w2, len)	rc = WriteFile(fd, ptr, w2, &len, NULL)
 #else
 	ssize_t len;
 	size_t w2;
-#define DO_WRITE(rc, fd, ptr, w2, len)	len = write(fd, ptr, w2); rc = (len >= 0)
 #endif
 
 	/* Do the lock/unlock of the reader mutex before starting the
@@ -11552,8 +11548,8 @@ mdb_env_copyfd(MDB_env *env, HANDLE fd)
 	return mdb_env_copyfd2(env, fd, 0);
 }
 
-int ESECT
-mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags)
+static int ESECT
+mdb_env_copy_open(MDB_env *env, const char *path, HANDLE *retfd)
 {
 	int rc;
 	MDB_name fname;
@@ -11564,11 +11560,23 @@ mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags)
 		rc = mdb_fopen(env, &fname, MDB_O_COPY, 0666, &newfd);
 		mdb_fname_destroy(fname);
 	}
-	if (rc == MDB_SUCCESS) {
-		rc = mdb_env_copyfd2(env, newfd, flags);
+	return rc;
+}
+
+int ESECT
+mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags)
+{
+	HANDLE newfd = INVALID_HANDLE_VALUE;
+	int rc;
+
+	rc = mdb_env_copy_open(env, path, &newfd);
+	if (rc)
+		return rc;
+	rc = mdb_env_copyfd2(env, newfd, flags);
+
+	if (newfd != INVALID_HANDLE_VALUE)
 		if (close(newfd) < 0 && rc == MDB_SUCCESS)
 			rc = ErrCode();
-	}
 	return rc;
 }
 
@@ -11578,6 +11586,173 @@ mdb_env_copy(MDB_env *env, const char *path)
 	return mdb_env_copy2(env, path, 0);
 }
 
+int ESECT
+mdb_env_incr_dumpfd(MDB_env *env, HANDLE fd, size_t txnid)
+{
+	int rc;
+	MDB_page *mp, *mend;
+	MDB_txn *txn;
+	size_t wsize;
+	char *buf = NULL;
+#ifdef _WIN32
+	DWORD len, w2;
+#else
+	ssize_t len;
+	size_t w2;
+#endif
+
+	MEMALIGN(rc, buf, env->me_psize, 2*env->me_psize);
+	if (rc)
+		return rc;
+
+	/* Do the lock/unlock of the reader mutex before starting the
+	 * write txn.  Otherwise other read txns could block writers.
+	 */
+	rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
+	if (rc) {
+		ALIGNED_FREE(buf);
+		return rc;
+	}
+
+	if (env->me_txns) {
+		/* We must start the actual read txn after blocking writers */
+		mdb_txn_reset(txn);
+
+		/* Temporarily block writers until we snapshot the meta pages */
+		if (LOCK_MUTEX(rc, env, env->me_wmutex))
+			goto leave;
+
+		rc = mdb_txn_renew0(txn);
+		if (rc) {
+			UNLOCK_MUTEX(env->me_wmutex);
+			goto leave;
+		}
+	}
+
+	memcpy(buf, env->me_map, env->me_psize*2);
+
+	if (env->me_txns)
+		UNLOCK_MUTEX(env->me_wmutex);
+
+	mp = (MDB_page *)buf;
+	if (mp->mp_txnid > txnid) {
+		DO_WRITE(rc, fd, mp, env->me_psize, len);
+		if (!rc) {
+			rc = ErrCode();
+			goto leave;
+		}
+	}
+	mp = (MDB_page *)((char *)mp + env->me_psize);
+	if (mp->mp_txnid > txnid) {
+		DO_WRITE(rc, fd, mp, env->me_psize, len);
+		if (!rc) {
+			rc = ErrCode();
+			goto leave;
+		}
+	}
+	ALIGNED_FREE(buf);
+	buf = NULL;
+
+	mp = (MDB_page *)((char *)env->me_map + 2*env->me_psize);
+	mend = (MDB_page *)((char *)env->me_map + txn->mt_next_pgno * env->me_psize);
+	while (mp < mend) {
+		wsize = env->me_psize;
+		if (IS_OVERFLOW(mp))
+			wsize *= mp->mp_pages;
+		if (mp->mp_txnid > txnid) {
+			char *ptr = (char *)mp;
+			w2 = wsize;
+			while (w2 > 0) {
+				DO_WRITE(rc, fd, ptr, w2, len);
+				if (!rc) {
+					rc = ErrCode();
+					goto leave;
+				} else if (len > 0) {
+					rc = MDB_SUCCESS;
+					ptr += len;
+					w2 -= len;
+					continue;
+				} else {
+					rc = EIO;
+					goto leave;
+				}
+			}
+		}
+		mp = (MDB_page *)((char *)mp + wsize);
+	}
+
+leave:
+	mdb_txn_abort(txn);
+	if (buf != NULL)
+		ALIGNED_FREE(buf);
+	return rc;
+}
+
+int ESECT
+mdb_env_incr_dump(MDB_env *env, const char *path, size_t txnid)
+{
+	HANDLE newfd = INVALID_HANDLE_VALUE;
+	int rc;
+
+	/* Output is just a plain file, not an environment */
+	env->me_flags |= MDB_NOSUBDIR;
+
+	rc = mdb_env_copy_open(env, path, &newfd);
+	if (rc)
+		return rc;
+	rc = mdb_env_incr_dumpfd(env, newfd, txnid);
+
+	if (newfd != INVALID_HANDLE_VALUE)
+		if (close(newfd) < 0 && rc == MDB_SUCCESS)
+			rc = ErrCode();
+	return rc;
+}
+
+int ESECT
+mdb_env_incr_loadfd(MDB_env *env, HANDLE fd)
+{
+	size_t rsize;
+	ssize_t rlen;
+	char buf[PAGEHDRSZ], *ptr;
+	MDB_page *rp = (MDB_page *)buf, *mp;
+
+	if (!(env->me_flags & MDB_WRITEMAP))
+		return EINVAL;
+
+	for (;;) {
+#ifdef _WIN32
+		int rc = ReadFile(fd, buf, sizeof(buf), &rlen, NULL) ? (int)rlen : -1;
+		if (rc == -1 && ErrCode() == ERROR_HANDLE_EOF)
+			rc = 0;
+#else
+		rlen = read(fd, buf, sizeof(buf));
+#endif
+		if (rlen != sizeof(buf))
+			break;
+		rsize = env->me_psize;
+		if (IS_OVERFLOW(rp))
+			rsize *= rp->mp_pages;
+		rsize -= rlen;
+		mp = (MDB_page *)(env->me_map + rp->mp_pgno * env->me_psize);
+		ptr = METADATA(mp);
+		memcpy(mp, rp, sizeof(buf));
+		while (rsize > 0) {
+#ifdef _WIN32
+			rc = ReadFile(fd, ptr, rsize, &rlen, NULL) ? (int)rlen : -1;
+			if (rc == -1)
+				rlen = -1;
+#else
+			rlen = read(fd, ptr, rsize);
+#endif
+			if (rlen == -1)
+				return ErrCode();
+			ptr += rlen;
+			rsize -= rlen;
+		}
+	}
+	return MDB_SUCCESS;
+}
+
 int ESECT
 mdb_env_set_flags(MDB_env *env, unsigned int flag, int onoff)
 {
diff --git a/libraries/liblmdb/mdb_dump.1 b/libraries/liblmdb/mdb_dump.1
index 43b0b1c99a..068c037862 100644
--- a/libraries/liblmdb/mdb_dump.1
+++ b/libraries/liblmdb/mdb_dump.1
@@ -10,6 +10,8 @@ mdb_dump \- LMDB environment export tool
 [\c
 .BI \-f \ file\fR]
 [\c
+.BI \-i \ txnid\fR]
+[\c
 .BR \-l ]
 [\c
 .BR \-n ]
@@ -41,6 +43,12 @@ Write the library version number to the standard output, and exit.
 .BR \-f \ file
 Write to the specified file instead of to the standard output.
 .TP
+.BR \-i \ txnid
+Perform an incremental backup. Only pages that have been modified
+after the given transaction ID will be written in the dump.
+
+Note: This is a raw binary dump of the database pages, not the portable output format.
+.TP
 .BR \-l
 List the databases stored in the environment. Just the
 names will be listed, no data will be output.
diff --git a/libraries/liblmdb/mdb_dump.c b/libraries/liblmdb/mdb_dump.c
index fac85d0f8c..cefc664073 100644
--- a/libraries/liblmdb/mdb_dump.c
+++ b/libraries/liblmdb/mdb_dump.c
@@ -20,6 +20,13 @@
 #include <signal.h>
 #include "lmdb.h"
 
+#ifdef _WIN32
+#include <windows.h>
+#define	MDB_STDOUT	GetStdHandle(STD_OUTPUT_HANDLE)
+#else
+#define	MDB_STDOUT	1
+#endif
+
 #define Yu	MDB_PRIy(u)
 
 #define PRINT	1
@@ -153,7 +160,7 @@ static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name)
 
 static void usage(char *prog)
 {
-	fprintf(stderr, "usage: %s [-V] [-f output] [-l] [-n] [-p] [-v] [-m module [-w password]] [-a|-s subdb] dbpath\n", prog);
+	fprintf(stderr, "usage: %s [-V] [-f output] [-i txnid] [-l] [-n] [-p] [-v] [-m module [-w password]] [-a|-s subdb] dbpath\n", prog);
 	exit(EXIT_FAILURE);
 }
 
@@ -164,8 +171,9 @@ int main(int argc, char *argv[])
 	MDB_txn *txn;
 	MDB_dbi dbi;
 	char *prog = argv[0];
-	char *envname;
+	char *envname, *outname = NULL;
 	char *subname = NULL;
+	size_t txnid = 0;
 	int alldbs = 0, envflags = 0, list = 0;
 	char *module = NULL, *password = NULL, *errmsg;
 	void *mlm = NULL;
@@ -179,11 +187,12 @@ int main(int argc, char *argv[])
 	 * -n: use NOSUBDIR flag on env_open
 	 * -p: use printable characters
 	 * -f: write to file instead of stdout
+	 * -i: do incremental dump from txnid
 	 * -v: use previous snapshot
 	 * -V: print version and exit
 	 * (default) dump only the main DB
 	 */
-	while ((i = getopt(argc, argv, "af:lm:nps:vw:V")) != EOF) {
+	while ((i = getopt(argc, argv, "af:i:lm:nps:vw:V")) != EOF) {
 		switch(i) {
 		case 'V':
 			printf("%s\n", MDB_VERSION_STRING);
@@ -198,8 +207,11 @@ int main(int argc, char *argv[])
 			alldbs++;
 			break;
 		case 'f':
-			if (freopen(optarg, "w", stdout) == NULL) {
-				fprintf(stderr, "%s: %s: reopen: %s\n",
+			outname = optarg;
+			break;
+		case 'i':
+			if (sscanf(optarg, "%" Yu "i", &txnid) != 1 || !txnid) {
+				fprintf(stderr, "%s: %s: invalid txnid: %s\n",
 					prog, optarg, strerror(errno));
 				exit(EXIT_FAILURE);
 			}
@@ -268,6 +280,22 @@ int main(int argc, char *argv[])
 		goto env_close;
 	}
 
+	if (txnid) {
+		if (outname)
+			rc = mdb_env_incr_dump(env, outname, txnid);
+		else
+			rc = mdb_env_incr_dumpfd(env, MDB_STDOUT, txnid);
+		if (rc)
+			fprintf(stderr, "mdb_env_incr_dump failed, error %d %s\n", rc, mdb_strerror(rc));
+		goto env_close;
+	}
+
+	if (outname && freopen(outname, "w", stdout) == NULL) {
+		fprintf(stderr, "%s: %s: reopen: %s\n",
+			prog, outname, strerror(errno));
+		exit(EXIT_FAILURE);
+	}
+
 	rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
 	if (rc) {
 		fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
diff --git a/libraries/liblmdb/mdb_load.1 b/libraries/liblmdb/mdb_load.1
index c6f9554e26..b99a80397f 100644
--- a/libraries/liblmdb/mdb_load.1
+++ b/libraries/liblmdb/mdb_load.1
@@ -12,6 +12,8 @@ mdb_load \- LMDB environment import tool
 [\c
 .BI \-f \ file\fR]
 [\c
+.BR \-i ]
+[\c
 .BR \-n ]
 [\c
 .BI \-m \ module
@@ -55,6 +57,9 @@ on a database that uses custom compare functions.
 .BR \-f \ file
 Read from the specified file instead of from the standard input.
 .TP
+.BR \-i
+Load an incremental backup.
+.TP
 .BR \-n
 Load an LMDB database which does not use subdirectories.
 .TP
diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c
index b0f06b8f37..d3d736a411 100644
--- a/libraries/liblmdb/mdb_load.c
+++ b/libraries/liblmdb/mdb_load.c
@@ -19,6 +19,13 @@
 #include <unistd.h>
 #include "lmdb.h"
 
+#ifdef _WIN32
+#include <windows.h>
+#define	MDB_STDIN	GetStdHandle(STD_INPUT_HANDLE)
+#else
+#define	MDB_STDIN	0
+#endif
+
 #define PRINT	1
 #define NOHDR	2
 static int mode;
@@ -295,7 +302,7 @@ badend:
 
 static void usage(void)
 {
-	fprintf(stderr, "usage: %s [-V] [-a] [-f input] [-n] [-m module [-w password]] [-s name] [-N] [-Q] [-T] dbpath\n", prog);
+	fprintf(stderr, "usage: %s [-V] [-a] [-f input] [-i] [-n] [-m module [-w password]] [-s name] [-N] [-Q] [-T] dbpath\n", prog);
 	exit(EXIT_FAILURE);
 }
 
@@ -313,7 +320,7 @@ int main(int argc, char *argv[])
 	MDB_dbi dbi;
 	char *envname;
 	int envflags = MDB_NOSYNC, putflags = 0;
-	int dohdr = 0, append = 0;
+	int dohdr = 0, append = 0, incr = 0;
 	MDB_val prevk;
 	char *module = NULL, *password = NULL, *errmsg;
 	void *mlm = NULL;
@@ -326,6 +333,8 @@ int main(int argc, char *argv[])
 
 	/* -a: append records in input order
 	 * -f: load file instead of stdin
+	 * -i: load an incremental dump
+	 * -m: dynamically load a module
 	 * -n: use NOSUBDIR flag on env_open
 	 * -s: load into named subDB
 	 * -N: use NOOVERWRITE on puts
@@ -333,7 +342,7 @@ int main(int argc, char *argv[])
 	 * -T: read plaintext
 	 * -V: print version and exit
 	 */
-	while ((i = getopt(argc, argv, "af:m:ns:w:NQTV")) != EOF) {
+	while ((i = getopt(argc, argv, "af:im:ns:w:NQTV")) != EOF) {
 		switch(i) {
 		case 'V':
 			printf("%s\n", MDB_VERSION_STRING);
@@ -349,6 +358,9 @@ int main(int argc, char *argv[])
 				exit(EXIT_FAILURE);
 			}
 			break;
+		case 'i':
+			incr = 1;
+			break;
 		case 'n':
 			envflags |= MDB_NOSUBDIR;
 			break;
@@ -377,6 +389,7 @@ int main(int argc, char *argv[])
 
 	if (optind != argc - 1)
 		usage();
+	envname = argv[optind];
 
 	dbuf.mv_size = 4096;
 	dbuf.mv_data = malloc(dbuf.mv_size);
@@ -384,7 +397,6 @@ int main(int argc, char *argv[])
 	if (!(mode & NOHDR))
 		readhdr();
 
-	envname = argv[optind];
 	rc = mdb_env_create(&env);
 	if (rc) {
 		fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc));
@@ -400,6 +412,17 @@ int main(int argc, char *argv[])
 		mdb_modsetup(env, mcf, password);
 	}
 
+	if (incr) {
+		envflags |= MDB_WRITEMAP;
+		rc = mdb_env_open(env, envname, envflags, 0664);
+		if (rc)
+			goto openfail;
+		rc = mdb_env_incr_loadfd(env, MDB_STDIN);
+		if (rc)
+			fprintf(stderr, "mdb_env_incr_load failed, error %d %s\n", rc, mdb_strerror(rc));
+		goto env_close;
+	}
+
 	mdb_env_set_maxdbs(env, 2);
 
 	if (info.me_maxreaders)
@@ -416,6 +439,7 @@ int main(int argc, char *argv[])
 
 	rc = mdb_env_open(env, envname, envflags, 0664);
 	if (rc) {
+openfail:
 		fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
 		goto env_close;
 	}

From e11d5a0529033a86520a54f870d85c40335f8e6e Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 22 Apr 2026 09:34:27 +0100
Subject: [PATCH 476/504] ITS#9223 LMDB: cleanup prev commit

Fix age check of meta pages.
Use MAX_WRITE consistently in copy functions.
---
 libraries/liblmdb/mdb.c | 58 +++++++++++++++++++++++------------------
 1 file changed, 32 insertions(+), 26 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 82e5863790..5e43747069 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -11045,7 +11045,7 @@ typedef struct mdb_copy {
 	char *mc_wbuf[2];
 	char *mc_over[2];
 	size_t mc_wlen[2];
-	size_t mc_olen[2];
+	mdb_size_t mc_olen[2];
 	pgno_t mc_next_pgno;
 	HANDLE mc_fd;
 	int mc_toggle;			/**< Buffer number in provider */
@@ -11069,11 +11069,12 @@ mdb_env_copythr(void *arg)
 	mdb_copy *my = arg;
 	char *ptr;
 	int toggle = 0, rc;
-	size_t wsize;
+	mdb_size_t wsize;
 #ifdef _WIN32
-	DWORD len;
+	DWORD len, w2;
 #else
-	int len;
+	ssize_t len;
+	size_t w2;
 #ifdef SIGPIPE
 	sigset_t set;
 	sigemptyset(&set);
@@ -11094,7 +11095,8 @@ mdb_env_copythr(void *arg)
 again:
 		rc = MDB_SUCCESS;
 		while (wsize > 0 && !my->mc_error) {
-			DO_WRITE(rc, my->mc_fd, ptr, wsize, len);
+			w2 = (wsize > MAX_WRITE) ? MAX_WRITE : wsize;
+			DO_WRITE(rc, my->mc_fd, ptr, w2, len);
 			if (!rc) {
 				rc = ErrCode();
 #if defined(SIGPIPE) && !defined(_WIN32)
@@ -11509,10 +11511,7 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd)
 	}
 	wsize = w3 - wsize;
 	while (wsize > 0) {
-		if (wsize > MAX_WRITE)
-			w2 = MAX_WRITE;
-		else
-			w2 = wsize;
+		w2 = (wsize > MAX_WRITE) ? MAX_WRITE : wsize;
 		DO_WRITE(rc, fd, ptr, w2, len);
 		if (!rc) {
 			rc = ErrCode();
@@ -11592,7 +11591,7 @@ mdb_env_incr_dumpfd(MDB_env *env, HANDLE fd, size_t txnid)
 	int rc;
 	MDB_page *mp, *mend;
 	MDB_txn *txn;
-	size_t wsize;
+	mdb_size_t wsize;
 	char *buf = NULL;
 #ifdef _WIN32
 	DWORD len, w2;
@@ -11634,21 +11633,27 @@ mdb_env_incr_dumpfd(MDB_env *env, HANDLE fd, size_t txnid)
 	if (env->me_txns)
 		UNLOCK_MUTEX(env->me_wmutex);
 
-	mp = (MDB_page *)buf;
-	if (mp->mp_txnid > txnid) {
-		DO_WRITE(rc, fd, mp, env->me_psize, len);
-		if (!rc) {
-			rc = ErrCode();
-			goto leave;
+	{
+		MDB_meta *mm;
+		mp = (MDB_page *)buf;
+		mm = METADATA(mp);
+		if (mm->mm_txnid > txnid) {
+			DO_WRITE(rc, fd, mp, env->me_psize, len);
+			if (!rc) {
+				rc = ErrCode();
+				goto leave;
+			}
 		}
-	}
-	mp = (MDB_page *)((char *)mp + env->me_psize);
-	if (mp->mp_txnid > txnid) {
-		DO_WRITE(rc, fd, mp, env->me_psize, len);
-		if (!rc) {
-			rc = ErrCode();
-			goto leave;
+		mp = (MDB_page *)((char *)mp + env->me_psize);
+		mm = METADATA(mp);
+		if (mm->mm_txnid > txnid) {
+			DO_WRITE(rc, fd, mp, env->me_psize, len);
+			if (!rc) {
+				rc = ErrCode();
+				goto leave;
+			}
 		}
+
 	}
 	ALIGNED_FREE(buf);
 	buf = NULL;
@@ -11660,9 +11665,10 @@ mdb_env_incr_dumpfd(MDB_env *env, HANDLE fd, size_t txnid)
 		if (IS_OVERFLOW(mp))
 			wsize *= mp->mp_pages;
 		if (mp->mp_txnid > txnid) {
+			mdb_size_t w3 = wsize;
 			char *ptr = (char *)mp;
-			w2 = wsize;
-			while (w2 > 0) {
+			while (w3 > 0) {
+				w2 = (w3 > MAX_WRITE) ? MAX_WRITE : w3;
 				DO_WRITE(rc, fd, ptr, w2, len);
 				if (!rc) {
 					rc = ErrCode();
@@ -11670,7 +11676,7 @@ mdb_env_incr_dumpfd(MDB_env *env, HANDLE fd, size_t txnid)
 				} else if (len > 0) {
 					rc = MDB_SUCCESS;
 					ptr += len;
-					w2 -= len;
+					w3 -= len;
 					continue;
 				} else {
 					rc = EIO;

From 43670420426754bb16f4ac68798e9b58d3bff7e9 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 22 Apr 2026 11:26:00 +0100
Subject: [PATCH 477/504] ITS#8165 LMDB: add soname, lib version

---
 libraries/liblmdb/Makefile | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile
index e0c0532159..9cb18861b8 100644
--- a/libraries/liblmdb/Makefile
+++ b/libraries/liblmdb/Makefile
@@ -27,7 +27,11 @@ CFLAGS	= $(THREADS) $(OPT) $(W) $(XCFLAGS)
 LDFLAGS = $(THREADS)
 LDLIBS	= 
 SOLIBS	= 
+LIBVER	= 1
+ABIVER	= 0
+VEREXT	= $(LIBVER).$(ABIVER)
 SOEXT	= .so
+SOFULL	= $(SOEXT).$(VEREXT)
 LDL		= -ldl
 prefix	= /usr/local
 exec_prefix = $(prefix)
@@ -40,13 +44,14 @@ mandir = $(datarootdir)/man
 ########################################################################
 
 IHDRS	= lmdb.h
-ILIBS	= liblmdb.a liblmdb$(SOEXT)
+ILIBS	= liblmdb.a
+ILIBS2	= liblmdb$(SOFULL)
 IPROGS	= mdb_stat mdb_copy mdb_dump mdb_load mdb_drop
 IDOCS	= mdb_stat.1 mdb_copy.1 mdb_dump.1 mdb_load.1 mdb_drop.1
 PROGS	= $(IPROGS) mtest mtest2 mtest3 mtest4 mtest5
 RPROGS	= mtest_remap mtest_enc mtest_enc2
 
-all:	$(ILIBS) $(PROGS)
+all:	$(ILIBS) $(ILIBS2) $(PROGS)
 # Requires CPPFLAGS=-DMDB_VL32 and/or -DMDB_RPAGE_CACHE
 rall:	all $(RPROGS)
 
@@ -57,6 +62,7 @@ install: $(ILIBS) $(IPROGS) $(IHDRS)
 	mkdir -p $(DESTDIR)$(mandir)/man1
 	for f in $(IPROGS); do cp $$f $(DESTDIR)$(bindir); done
 	for f in $(ILIBS); do cp $$f $(DESTDIR)$(libdir); done
+	for f in $(ILIBS2); do cp $$f $(DESTDIR)$(libdir); ln -s $$f $(DESTDIR)$(libdir)/`basename -s .$(VEREXT) $$f`; done
 	for f in $(IHDRS); do cp $$f $(DESTDIR)$(includedir); done
 	for f in $(IDOCS); do cp $$f $(DESTDIR)$(mandir)/man1; done
 
@@ -70,9 +76,10 @@ test:	all
 liblmdb.a:	mdb.o midl.o module.o
 	$(AR) rs $@ mdb.o midl.o module.o
 
-liblmdb$(SOEXT):	mdb.lo midl.lo module.lo
+liblmdb$(SOFULL):	mdb.lo midl.lo module.lo
 #	$(CC) $(LDFLAGS) -pthread -shared -Wl,-Bsymbolic -o $@ mdb.o midl.o $(SOLIBS)
-	$(CC) $(LDFLAGS) -pthread -shared -o $@ mdb.lo midl.lo module.lo $(SOLIBS) $(LDL)
+	$(CC) $(LDFLAGS) -shared -Wl,-soname,liblmdb$(SOEXT).$(LIBVER) -o $@ mdb.lo midl.lo module.lo $(SOLIBS) $(LDL)
+	rm -f liblmdb$(SOEXT); ln -s $@ liblmdb$(SOEXT)
 
 mdb_stat: mdb_stat.o liblmdb.a
 	$(CC) $(LDFLAGS) -o $@ $^ $(LDL)

From 0734803dba25040ac398ec445d1e57cb215d4e4f Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 22 Apr 2026 12:45:11 +0100
Subject: [PATCH 478/504] ITS#8165 more for prev commit

Fixup .so symlinks, fixup install, add MacOSX linker option
---
 libraries/liblmdb/Makefile | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile
index 9cb18861b8..badb1165d2 100644
--- a/libraries/liblmdb/Makefile
+++ b/libraries/liblmdb/Makefile
@@ -50,6 +50,10 @@ IPROGS	= mdb_stat mdb_copy mdb_dump mdb_load mdb_drop
 IDOCS	= mdb_stat.1 mdb_copy.1 mdb_dump.1 mdb_load.1 mdb_drop.1
 PROGS	= $(IPROGS) mtest mtest2 mtest3 mtest4 mtest5
 RPROGS	= mtest_remap mtest_enc mtest_enc2
+SOVER	= liblmdb$(SOEXT).$(LIBVER)
+VERSION_OPT	= -Wl,-soname,$(SOVER)
+# For MacOSX:
+#VERSION_OPT	= -Wl,-current_version,$(VEREXT)
 
 all:	$(ILIBS) $(ILIBS2) $(PROGS)
 # Requires CPPFLAGS=-DMDB_VL32 and/or -DMDB_RPAGE_CACHE
@@ -62,12 +66,16 @@ install: $(ILIBS) $(IPROGS) $(IHDRS)
 	mkdir -p $(DESTDIR)$(mandir)/man1
 	for f in $(IPROGS); do cp $$f $(DESTDIR)$(bindir); done
 	for f in $(ILIBS); do cp $$f $(DESTDIR)$(libdir); done
-	for f in $(ILIBS2); do cp $$f $(DESTDIR)$(libdir); ln -s $$f $(DESTDIR)$(libdir)/`basename -s .$(VEREXT) $$f`; done
+	for f in $(ILIBS2); do cp $$f $(DESTDIR)$(libdir); \
+		i=`basename -s .$(ABIVER) $$f`; rm -f $(DESTDIR)$(libdir)/$$i; \
+		ln -s $$f $(DESTDIR)$(libdir)/$$i; \
+		i=`basename -s .$(LIBVER) $$i`; rm -f $(DESTDIR)$(libdir)/$$i; \
+		ln -s $$f $(DESTDIR)$(libdir)/$$i; done
 	for f in $(IHDRS); do cp $$f $(DESTDIR)$(includedir); done
 	for f in $(IDOCS); do cp $$f $(DESTDIR)$(mandir)/man1; done
 
 clean:
-	rm -rf $(PROGS) $(RPROGS) *.[ao] *.[ls]o *~ testdb
+	rm -rf $(PROGS) $(RPROGS) *.[ao] *.[ls]o *.so.* *~ testdb
 
 test:	all
 	rm -rf testdb && mkdir testdb
@@ -78,8 +86,9 @@ liblmdb.a:	mdb.o midl.o module.o
 
 liblmdb$(SOFULL):	mdb.lo midl.lo module.lo
 #	$(CC) $(LDFLAGS) -pthread -shared -Wl,-Bsymbolic -o $@ mdb.o midl.o $(SOLIBS)
-	$(CC) $(LDFLAGS) -shared -Wl,-soname,liblmdb$(SOEXT).$(LIBVER) -o $@ mdb.lo midl.lo module.lo $(SOLIBS) $(LDL)
+	$(CC) $(LDFLAGS) -shared $(VERSION_OPT) -o $@ mdb.lo midl.lo module.lo $(SOLIBS) $(LDL)
 	rm -f liblmdb$(SOEXT); ln -s $@ liblmdb$(SOEXT)
+	rm -f $(SOVER); ln -s $@ $(SOVER)
 
 mdb_stat: mdb_stat.o liblmdb.a
 	$(CC) $(LDFLAGS) -o $@ $^ $(LDL)

From d0fcfd3d02321d04b56a23e6c86c80309f4ecef5 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 22 Apr 2026 13:24:29 +0100
Subject: [PATCH 479/504] ITS#8174 LMDB: fixes for mdb_drop(MAIN_DBI)

---
 libraries/liblmdb/lmdb.h |  2 ++
 libraries/liblmdb/mdb.c  | 30 ++++++++++++++++++++++++++++--
 2 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 6376a6b555..51131235cb 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -528,6 +528,8 @@ typedef enum MDB_cursor_op {
 #define MDB_TXN_PENDING		(-30775)
 	/** Environment can't rollback the last transaction */
 #define MDB_CANT_ROLLBACK	(-30774)
+	/** Can't drop main DBI while other DBIs are open */
+#define MDB_DBIS_BUSY	(-30773)
 	/** The last defined error code */
 #define MDB_LAST_ERRCODE	MDB_CANT_ROLLBACK
 /** @} */
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 5e43747069..6da71a4e7b 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1524,8 +1524,9 @@ struct MDB_txn {
 #define MDB_TXN_HAS_CHILD	0x10		/**< txn has an #MDB_txn.%mt_child */
 #define MDB_TXN_DIRTYNUM	0x20		/**< dirty list uses nump list */
 #define MDB_TXN_PREPARE		0x40		/**< prepare txn, don't fully commit */
+#define MDB_TXN_DROPPED		0x80		/**< main DBI has been dropped, env will reset */
 	/** most operations on the txn are currently illegal */
-#define MDB_TXN_BLOCKED		(MDB_TXN_FINISHED|MDB_TXN_ERROR|MDB_TXN_HAS_CHILD|MDB_TXN_PREPARE)
+#define MDB_TXN_BLOCKED		(MDB_TXN_FINISHED|MDB_TXN_ERROR|MDB_TXN_HAS_CHILD|MDB_TXN_PREPARE|MDB_TXN_DROPPED)
 /** @} */
 	unsigned int	mt_flags;		/**< @ref mdb_txn */
 	/** #dirty_list room: Array size - \#dirty pages visible to this txn.
@@ -1914,6 +1915,7 @@ static char *const mdb_errstr[] = {
 	"MDB_ENV_ENCRYPTION: Environment encryption mismatch",
 	"MDB_TXN_PENDING: Transaction already prepared, must abort or commit",
 	"MDB_CANT_ROLLBACK: Environment can't rollback last transaction",
+	"MDB_DBIS_BUSY: Can't drop main DBI while other DBIs are open",
 };
 
 char *
@@ -12232,11 +12234,35 @@ int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del)
 	if (TXN_DBI_CHANGED(txn, dbi))
 		return MDB_BAD_DBI;
 
+	MDB_TRACE(("%u, %d", dbi, del));
+
+	/* Dropping the main DBI empties/resets the entire environment.
+	 * Can't do it if any other DBIs are still open. Can only commit
+	 * or abort after this.
+	 */
+	if (dbi == MAIN_DBI) {
+		MDB_dbi i;
+		for (i = CORE_DBS; i<txn->mt_numdbs; i++) {
+			if (txn->mt_dbflags[i] & DB_VALID)
+				return MDB_DBIS_BUSY;
+		}
+		for (i = 0; i<CORE_DBS; i++) {
+			txn->mt_dbflags[i] |= DB_DIRTY;
+			txn->mt_dbs[i].md_depth = 0;
+			txn->mt_dbs[i].md_branch_pages = 0;
+			txn->mt_dbs[i].md_leaf_pages = 0;
+			txn->mt_dbs[i].md_overflow_pages = 0;
+			txn->mt_dbs[i].md_entries = 0;
+			txn->mt_dbs[i].md_root = P_INVALID;
+		}
+		txn->mt_flags |= MDB_TXN_DIRTY|MDB_TXN_DROPPED;
+		return MDB_SUCCESS;
+	}
+
 	rc = mdb_cursor_open(txn, dbi, &mc);
 	if (rc)
 		return rc;
 
-	MDB_TRACE(("%u, %d", dbi, del));
 	rc = mdb_drop0(mc, mc->mc_db->md_flags & MDB_DUPSORT);
 	/* Invalidate the dropped DB's cursors */
 	for (m2 = txn->mt_cursors[dbi]; m2; m2 = m2->mc_next)

From a0cb775d89f93dab34ba36d2e022c16047ff57ea Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 22 Apr 2026 17:43:41 +0100
Subject: [PATCH 480/504] mdb_dump: fix symbol clash on Windows

rename byte() to dobyte()
---
 libraries/liblmdb/mdb_dump.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/mdb_dump.c b/libraries/liblmdb/mdb_dump.c
index cefc664073..7a79eaec64 100644
--- a/libraries/liblmdb/mdb_dump.c
+++ b/libraries/liblmdb/mdb_dump.c
@@ -83,7 +83,7 @@ static void text(MDB_val *v)
 	putchar('\n');
 }
 
-static void byte(MDB_val *v)
+static void dobyte(MDB_val *v)
 {
 	unsigned char *c, *end;
 
@@ -147,8 +147,8 @@ static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name)
 			text(&key);
 			text(&data);
 		} else {
-			byte(&key);
-			byte(&data);
+			dobyte(&key);
+			dobyte(&data);
 		}
 	}
 	printf("DATA=END\n");

From 63698faf2c159136443e5e3c07541201fca5ee71 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 22 Apr 2026 19:22:56 +0100
Subject: [PATCH 481/504] ITS#9027 LMDB: expose address of memory map

Returned in MDB_envinfo.
---
 libraries/liblmdb/lmdb.h | 2 +-
 libraries/liblmdb/mdb.c  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 51131235cb..bb692e2c0b 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -547,7 +547,7 @@ typedef struct MDB_stat {
 
 /** @brief Information about the environment */
 typedef struct MDB_envinfo {
-	void	*me_mapaddr;			/**< Address of map, if fixed */
+	void	*me_mapaddr;			/**< Address of map */
 	mdb_size_t	me_mapsize;				/**< Size of the data memory map */
 	mdb_size_t	me_last_pgno;			/**< ID of the last used page */
 	mdb_size_t	me_last_txnid;			/**< ID of the last committed transaction */
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 6da71a4e7b..c8e057463e 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -11920,7 +11920,7 @@ mdb_env_info(MDB_env *env, MDB_envinfo *arg)
 		return EINVAL;
 
 	meta = mdb_env_pick_meta(env);
-	arg->me_mapaddr = meta->mm_address;
+	arg->me_mapaddr = meta->mm_address ? meta->mm_address : env->me_map;
 	arg->me_last_pgno = meta->mm_last_pg;
 	arg->me_last_txnid = meta->mm_txnid;
 

From 81697bc2200c1e8a55689f99b7d4763e999abaac Mon Sep 17 00:00:00 2001
From: Lorenz Bauer <lmb@cloudflare.com>
Date: Wed, 15 Feb 2017 18:58:05 +0000
Subject: [PATCH 482/504] ITS#8590 LMDB: Use F_SETNOSIGPIPE on OS X

It seems like OS X delivers SIGPIPE to the whole process, instead of the generating thread,
as on other UNIXes. Therefore the current code sometimes works, but mostly doesn't, since
the call to sigwait doesn't happen quickly enough.

Instead of masking SIGPIPE, we use an OS X specific fcntl to disable SIGPIPE for that particular fd.
---
 libraries/liblmdb/mdb.c | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index c8e057463e..dbb1504683 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -11077,7 +11077,14 @@ mdb_env_copythr(void *arg)
 #else
 	ssize_t len;
 	size_t w2;
-#ifdef SIGPIPE
+#ifdef F_SETNOSIGPIPE
+	/* OS X delivers SIGPIPE to the whole process, not the thread that caused it.
+	 * Disable SIGPIPE using platform specific fcntl.
+	 */
+	int enabled = 1;
+	if ((rc = fcntl(my->mc_fd, F_SETNOSIGPIPE, &enabled)) != 0)
+		my->mc_error = errno;
+#elif defined SIGPIPE
 	sigset_t set;
 	sigemptyset(&set);
 	sigaddset(&set, SIGPIPE);
@@ -11101,15 +11108,6 @@ again:
 			DO_WRITE(rc, my->mc_fd, ptr, w2, len);
 			if (!rc) {
 				rc = ErrCode();
-#if defined(SIGPIPE) && !defined(_WIN32)
-				if (rc == EPIPE) {
-					/* Collect the pending SIGPIPE, otherwise at least OS X
-					 * gives it to the process on thread-exit (ITS#8504).
-					 */
-					int tmp;
-					sigwait(&set, &tmp);
-				}
-#endif
 				break;
 			} else if (len > 0) {
 				rc = MDB_SUCCESS;

From c607a4b98a14d38b2364a04e1f28e6c6910fb1ba Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 23 Apr 2026 10:33:01 +0100
Subject: [PATCH 483/504] ITS#9223 more cleanup for prev commit

---
 libraries/liblmdb/mdb.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index dbb1504683..1f233cb7e6 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -11552,11 +11552,10 @@ mdb_env_copy_open(MDB_env *env, const char *path, HANDLE *retfd)
 {
 	int rc;
 	MDB_name fname;
-	HANDLE newfd = INVALID_HANDLE_VALUE;
 
 	rc = mdb_fname_init(path, env->me_flags | MDB_NOLOCK, &fname);
 	if (rc == MDB_SUCCESS) {
-		rc = mdb_fopen(env, &fname, MDB_O_COPY, 0666, &newfd);
+		rc = mdb_fopen(env, &fname, MDB_O_COPY, 0666, retfd);
 		mdb_fname_destroy(fname);
 	}
 	return rc;

From 9aa3ac00567db8d5222abea46325a8b882937350 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 23 Apr 2026 11:44:34 +0100
Subject: [PATCH 484/504] ITS#9619 LMDB: fix mdb_env_copy2(MDB_COMPACT) for
 overflow pages

Wasn't resetting txnid in page header
---
 libraries/liblmdb/mdb.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 1f233cb7e6..a5c4cde1cb 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -11238,6 +11238,7 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags)
 						}
 						mo = (MDB_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]);
 						memcpy(mo, omp, my->mc_env->me_psize);
+						mo->mp_txnid = 1;
 						ovp.op_pgno = my->mc_next_pgno;
 						ovp.op_txnid = 1;
 						memcpy(NODEDATA(ni), &ovp, sizeof(ovp));

From cf52db2bee52796bb4140b32679f4723a0d7d44f Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Thu, 23 Apr 2026 18:55:06 +0100
Subject: [PATCH 485/504] ITS#8579 LMDB: all descriptors should have O_CLOEXEC

There's nothing useful that can be done with them if they're kept around.
---
 libraries/liblmdb/mdb.c | 28 ++++++++++++----------------
 1 file changed, 12 insertions(+), 16 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index a5c4cde1cb..e5cc664796 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5301,15 +5301,15 @@ enum mdb_fopen_type {
 	MDB_O_RDONLY, MDB_O_RDWR, MDB_O_OVERLAPPED, MDB_O_META, MDB_O_COPY, MDB_O_LOCKS
 #else
 	/* A comment in mdb_fopen() explains some O_* flag choices. */
-	MDB_O_RDONLY= O_RDONLY,                            /**< for RDONLY me_fd */
-	MDB_O_RDWR  = O_RDWR  |O_CREAT,                    /**< for me_fd */
-	MDB_O_META  = O_WRONLY|MDB_DSYNC     |MDB_CLOEXEC, /**< for me_mfd */
-	MDB_O_COPY  = O_WRONLY|O_CREAT|O_EXCL|MDB_CLOEXEC, /**< for #mdb_env_copy() */
+	MDB_O_RDONLY= O_RDONLY               , /**< for RDONLY me_fd */
+	MDB_O_RDWR  = O_RDWR  |O_CREAT       , /**< for me_fd */
+	MDB_O_META  = O_WRONLY|MDB_DSYNC     , /**< for me_mfd */
+	MDB_O_COPY  = O_WRONLY|O_CREAT|O_EXCL, /**< for #mdb_env_copy() */
 	/** Bitmask for open() flags in enum #mdb_fopen_type.  The other bits
 	 * distinguish otherwise-equal MDB_O_* constants from each other.
 	 */
-	MDB_O_MASK  = MDB_O_RDWR|MDB_CLOEXEC | MDB_O_RDONLY|MDB_O_META|MDB_O_COPY,
-	MDB_O_LOCKS = MDB_O_RDWR|MDB_CLOEXEC | ((MDB_O_MASK+1) & ~MDB_O_MASK) /**< for me_lfd */
+	MDB_O_MASK  = MDB_O_RDWR| MDB_O_RDONLY|MDB_O_META|MDB_O_COPY,
+	MDB_O_LOCKS = MDB_O_RDWR| ((MDB_O_MASK+1) & ~MDB_O_MASK) /**< for me_lfd */
 #endif
 };
 
@@ -5346,11 +5346,8 @@ mdb_fopen(const MDB_env *env, MDB_name *fname,
 	 * With MDB_O_COPY we do not want the OS to cache the writes, since
 	 * the source data is already in the OS cache.
 	 *
-	 * The lockfile needs FD_CLOEXEC (close file descriptor on exec*())
-	 * to avoid the flock() issues noted under Caveats in lmdb.h.
-	 * Also set it for other filehandles which the user cannot get at
-	 * and close himself, which he may need after fork().  I.e. all but
-	 * me_fd, which programs do use via mdb_env_get_fd().
+	 * All files are opened with MDB_CLOEXEC to prevent leaking across
+	 * an exec().
 	 */
 
 #ifdef _WIN32
@@ -5383,17 +5380,16 @@ mdb_fopen(const MDB_env *env, MDB_name *fname,
 	}
 	fd = CreateFileW(fname->mn_val, acc, share, NULL, disp, attrs, NULL);
 #else
-	fd = open(fname->mn_val, which & MDB_O_MASK, mode);
+	fd = open(fname->mn_val, (which & MDB_O_MASK)|MDB_CLOEXEC, mode);
 #endif
 
 	if (fd == INVALID_HANDLE_VALUE)
 		rc = ErrCode();
 #ifndef _WIN32
 	else {
-		if (which != MDB_O_RDONLY && which != MDB_O_RDWR) {
-			/* Set CLOEXEC if we could not pass it to open() */
-			if (!MDB_CLOEXEC && (flags = fcntl(fd, F_GETFD)) != -1)
-				(void) fcntl(fd, F_SETFD, flags | FD_CLOEXEC);
+		/* Set CLOEXEC if we could not pass it to open() */
+		if (!MDB_CLOEXEC && (flags = fcntl(fd, F_GETFD)) != -1) {
+			(void) fcntl(fd, F_SETFD, flags | FD_CLOEXEC);
 		}
 		if (which == MDB_O_COPY && env->me_psize >= env->me_os_psize) {
 			/* This may require buffer alignment.  There is no portable

From bf5c4e0f2302c07a1a5552143be5a353a1fb3fe9 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Fri, 24 Apr 2026 16:05:40 +0100
Subject: [PATCH 486/504] ITS#8192 LMDB: define new error codes to avoid errno
 abuse

And avoid confusion with Windows error codes
---
 libraries/liblmdb/lmdb.h | 10 ++++++-
 libraries/liblmdb/mdb.c  | 56 +++++++++++++++++++---------------------
 2 files changed, 35 insertions(+), 31 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index bb692e2c0b..454532b618 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -530,8 +530,16 @@ typedef enum MDB_cursor_op {
 #define MDB_CANT_ROLLBACK	(-30774)
 	/** Can't drop main DBI while other DBIs are open */
 #define MDB_DBIS_BUSY	(-30773)
+	/** Write was incomplete */
+#define MDB_SHORT_WRITE	(-30772)
+	/** Env is busy, can't use previous snapshot */
+#define MDB_ENV_BUSY	(-30771)
+	/** Env or txn is read-only, can't write */
+#define MDB_IS_READONLY	(-30770)
+	/** Requested map address is unavailable */
+#define MDB_ADDR_BUSY
 	/** The last defined error code */
-#define MDB_LAST_ERRCODE	MDB_CANT_ROLLBACK
+#define MDB_LAST_ERRCODE	MDB_ADDR_BUSY
 /** @} */
 
 /** @brief Statistics for a database in the environment */
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index e5cc664796..03f0a19ea1 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -330,6 +330,7 @@ union semun {
 #endif
 
 /* Internal error codes, not exposed outside liblmdb */
+#define	MDB_NO_META		(MDB_LAST_ERRCODE + 9)
 #define	MDB_NO_ROOT		(MDB_LAST_ERRCODE + 10)
 #ifdef _WIN32
 #define MDB_OWNERDEAD	((int) WAIT_ABANDONED)
@@ -1916,6 +1917,10 @@ static char *const mdb_errstr[] = {
 	"MDB_TXN_PENDING: Transaction already prepared, must abort or commit",
 	"MDB_CANT_ROLLBACK: Environment can't rollback last transaction",
 	"MDB_DBIS_BUSY: Can't drop main DBI while other DBIs are open",
+	"MDB_SHORT_WRITE: Fewer bytes were written than requested",
+	"MDB_ENV_BUSY: Environment is busy, can't use previous snapshot",
+	"MDB_IS_READONLY: Can't write in readonly txn or environment",
+	"MDB_ADDR_BUSY: Requested map address is unavailable",
 };
 
 char *
@@ -1946,13 +1951,8 @@ mdb_strerror(int err)
 	 * have used LMDB-specific error codes for everything.
 	 */
 	switch(err) {
-	case ENOENT:	/* 2, FILE_NOT_FOUND */
-	case EIO:		/* 5, ACCESS_DENIED */
 	case ENOMEM:	/* 12, INVALID_ACCESS */
-	case EACCES:	/* 13, INVALID_DATA */
-	case EBUSY:		/* 16, CURRENT_DIRECTORY */
 	case EINVAL:	/* 22, BAD_COMMAND */
-	case ENOSPC:	/* 28, OUT_OF_PAPER */
 		return strerror(err);
 	default:
 		;
@@ -3104,7 +3104,7 @@ mdb_env_sync0(MDB_env *env, int force, pgno_t numpgs)
 {
 	int rc = 0;
 	if (env->me_flags & MDB_RDONLY)
-		return EACCES;
+		return MDB_IS_READONLY;
 	if (force || !(env->me_flags & MDB_NOSYNC)
 #ifdef _WIN32	/* Sync is normally achieved in Windows by doing WRITE_THROUGH writes */
 		&& (env->me_flags & MDB_WRITEMAP)
@@ -3485,7 +3485,7 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 	flags |= env->me_flags & MDB_WRITEMAP;
 
 	if (env->me_flags & MDB_RDONLY & ~flags) /* write txn in RDONLY env */
-		return EACCES;
+		return MDB_IS_READONLY;
 
 	if (parent) {
 		/* Nested transactions:
@@ -4248,7 +4248,7 @@ bad_write:
 							goto retry_write;
 						DPRINTF(("Write error: %s", strerror(rc)));
 					} else {
-						rc = EIO; /* TODO: Use which error code? */
+						rc = MDB_SHORT_WRITE;
 						DPUTS("short write, filesystem full?");
 					}
 				}
@@ -4677,7 +4677,7 @@ mdb_env_read_header(MDB_env *env, int prev, MDB_meta *meta)
 		p = (MDB_page *)env->me_map;
 		for (i=0; i<NUM_METAS; i++) {
 			if (!F_ISSET(p->mp_flags, P_META))
-				return ENOENT;
+				return MDB_NO_META;
 			if (env->me_metas[i]->mm_magic != MDB_MAGIC)
 				return MDB_INVALID;
 			if (env->me_metas[i]->mm_version != MDB_DATA_VERSION)
@@ -4707,7 +4707,7 @@ mdb_env_read_header(MDB_env *env, int prev, MDB_meta *meta)
 #endif
 		if (rc != Size) {
 			if (rc == 0 && off == 0)
-				return ENOENT;
+				return MDB_NO_META;
 			rc = rc < 0 ? (int) ErrCode() : MDB_INVALID;
 			DPRINTF(("read: %s", mdb_strerror(rc)));
 			return rc;
@@ -4717,7 +4717,7 @@ mdb_env_read_header(MDB_env *env, int prev, MDB_meta *meta)
 
 		if (!F_ISSET(p->mp_flags, P_META)) {
 			if (env->me_flags & MDB_RAWPART)
-				return ENOENT;
+				return MDB_NO_META;
 			DPRINTF(("page %"Yu" not a meta page", p->mp_pgno));
 			return MDB_INVALID;
 		}
@@ -4822,7 +4822,7 @@ mdb_env_init_meta(MDB_env *env, MDB_meta *meta)
 	else if ((unsigned) len == psize * NUM_METAS)
 		rc = MDB_SUCCESS;
 	else
-		rc = ENOSPC;
+		rc = MDB_SHORT_WRITE;
 	free(p);
 	return rc;
 }
@@ -4925,7 +4925,7 @@ retry_write:
 	rc = pwrite(mfd, ptr, len, off);
 #endif
 	if (rc != len) {
-		rc = rc < 0 ? ErrCode() : EIO;
+		rc = rc < 0 ? ErrCode() : MDB_SHORT_WRITE;
 #ifndef _WIN32
 		if (rc == EINTR)
 			goto retry_write;
@@ -5128,7 +5128,7 @@ mdb_env_map(MDB_env *env, void *addr)
 	 * instead unmap existing pages to make room for the new map.
 	 */
 	if (addr && env->me_map != addr)
-		return EBUSY;	/* TODO: Make a new MDB_* error code? */
+		return MDB_ADDR_BUSY;
 
 	p = (MDB_page *)env->me_map;
 	env->me_metas[0] = METADATA(p);
@@ -5501,7 +5501,7 @@ mdb_env_open2(MDB_env *env, int prev)
 #endif
 
 	if ((i = mdb_env_read_header(env, prev, &meta)) != 0) {
-		if (i != ENOENT)
+		if (i != MDB_NO_META)
 			return i;
 		DPUTS("new mdbenv");
 		newenv = 1;
@@ -6077,10 +6077,6 @@ mdb_env_setup_locks(MDB_env *env, MDB_name *fname, int mode, int *excl)
 			rc = MDB_VERSION_MISMATCH;
 			goto fail;
 		}
-		rc = ErrCode();
-		if (rc && rc != EACCES && rc != EAGAIN) {
-			goto fail;
-		}
 #ifdef _WIN32
 		mdb_env_mname_init(env);
 		env->me_rmutex = OpenMutexA(SYNCHRONIZE, FALSE, MUTEXNAME(env, 'r'));
@@ -6297,7 +6293,7 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 		if (rc)
 			goto leave;
 		if ((flags & MDB_PREVSNAPSHOT) && !excl) {
-			rc = EAGAIN;
+			rc = MDB_ENV_BUSY;
 			goto leave;
 		}
 	}
@@ -8507,7 +8503,7 @@ _mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data,
 	flags &= ~MDB_NOSPILL;
 
 	if (mc->mc_txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_BLOCKED))
-		return (mc->mc_txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN;
+		return (mc->mc_txn->mt_flags & MDB_TXN_RDONLY) ? MDB_IS_READONLY : MDB_BAD_TXN;
 
 	if (key->mv_size-1 >= ENV_MAXKEY(env))
 		return MDB_BAD_VALSIZE;
@@ -9016,7 +9012,7 @@ _mdb_cursor_del(MDB_cursor *mc, unsigned int flags)
 	int rc;
 
 	if (mc->mc_txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_BLOCKED))
-		return (mc->mc_txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN;
+		return (mc->mc_txn->mt_flags & MDB_TXN_RDONLY) ? MDB_IS_READONLY : MDB_BAD_TXN;
 
 	if (!(mc->mc_flags & C_INITIALIZED))
 		return EINVAL;
@@ -10513,7 +10509,7 @@ mdb_del(MDB_txn *txn, MDB_dbi dbi,
 		return EINVAL;
 
 	if (txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_BLOCKED))
-		return (txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN;
+		return (txn->mt_flags & MDB_TXN_RDONLY) ? MDB_IS_READONLY : MDB_BAD_TXN;
 
 	if (!F_ISSET(txn->mt_dbs[dbi].md_flags, MDB_DUPSORT)) {
 		/* must ignore any data */
@@ -11017,7 +11013,7 @@ mdb_put(MDB_txn *txn, MDB_dbi dbi,
 		return EINVAL;
 
 	if (txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_BLOCKED))
-		return (txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN;
+		return (txn->mt_flags & MDB_TXN_RDONLY) ? MDB_IS_READONLY : MDB_BAD_TXN;
 
 	MDB_TRACE(("%p, %u, %"Z"u[%s], %"Z"u%s, %u",
 		txn, dbi, key ? key->mv_size:0, DKEY(key), data->mv_size, mdb_dval(txn, dbi, data, dbuf), flags));
@@ -11111,7 +11107,7 @@ again:
 				wsize -= len;
 				continue;
 			} else {
-				rc = EIO;
+				rc = MDB_SHORT_WRITE;
 				break;
 			}
 		}
@@ -11488,7 +11484,7 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd)
 			continue;
 		} else {
 			/* Non-blocking or async handles are not supported */
-			rc = EIO;
+			rc = MDB_SHORT_WRITE;
 			break;
 		}
 	}
@@ -11519,7 +11515,7 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd)
 			wsize -= len;
 			continue;
 		} else {
-			rc = EIO;
+			rc = MDB_SHORT_WRITE;
 			break;
 		}
 	}
@@ -11675,7 +11671,7 @@ mdb_env_incr_dumpfd(MDB_env *env, HANDLE fd, size_t txnid)
 					w3 -= len;
 					continue;
 				} else {
-					rc = EIO;
+					rc = MDB_SHORT_WRITE;
 					goto leave;
 				}
 			}
@@ -12022,7 +12018,7 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db
 		if (rc != MDB_NOTFOUND || !(flags & MDB_CREATE))
 			return rc;
 		if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY))
-			return EACCES;
+			return MDB_IS_READONLY;
 	}
 
 	/* Done here so we cannot fail after creating a new DB */
@@ -12223,7 +12219,7 @@ int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del)
 		return EINVAL;
 
 	if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY))
-		return EACCES;
+		return MDB_IS_READONLY;
 
 	if (TXN_DBI_CHANGED(txn, dbi))
 		return MDB_BAD_DBI;

From ba6e001801e043bafce44440c044a272fd7d6c95 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Fri, 24 Apr 2026 18:07:42 +0100
Subject: [PATCH 487/504] ITS#8192 fix prev commit

---
 libraries/liblmdb/lmdb.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 454532b618..da7b3c94a6 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -537,7 +537,7 @@ typedef enum MDB_cursor_op {
 	/** Env or txn is read-only, can't write */
 #define MDB_IS_READONLY	(-30770)
 	/** Requested map address is unavailable */
-#define MDB_ADDR_BUSY
+#define MDB_ADDR_BUSY	(-30769)
 	/** The last defined error code */
 #define MDB_LAST_ERRCODE	MDB_ADDR_BUSY
 /** @} */

From 4cd8e451679511c8fbeff68f8a97cbe7b1268b48 Mon Sep 17 00:00:00 2001
From: James Rouzier <rouzier@gmail.com>
Date: Wed, 1 Apr 2015 11:17:12 -0400
Subject: [PATCH 488/504] ITS#8250 LMDB: add option MDB_ROTXN_RESET flag for
 txn_begin

Create a readonly txn in a reset state, it cannot be used until
mdb_txn_renew() is called.
---
 libraries/liblmdb/lmdb.h |  9 +++++++++
 libraries/liblmdb/mdb.c  | 12 ++++++++++--
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index da7b3c94a6..3969ee81f7 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -372,6 +372,13 @@ typedef void (MDB_sum_func)(const MDB_val *src, MDB_val *dst, const MDB_val *key
 #define MDB_REMAP_CHUNKS	0x4000000
 /** @} */
 
+/** @defgroup	mdb_txn_begin	Transaction Flags
+ *	@{
+ */
+	/** don't initialize read-only txn; #mdb_txn_renew() must be called before first use */
+#define MDB_ROTXN_RESET	0x8000000
+/** @} */
+
 /**	@defgroup	mdb_dbi_open	Database Flags
  *	@{
  */
@@ -1143,6 +1150,8 @@ int mdb_env_set_checksum(MDB_env *env, MDB_sum_func *func, unsigned int size);
 	 *		Don't flush system buffers to disk when committing this transaction.
 	 *	<li>#MDB_NOMETASYNC
 	 *		Flush system buffers but omit metadata flush when committing this transaction.
+	 *	<li>#MDB_ROTXN_RESET
+	 *		Must call #mdb_txn_renew() before using this readonly transaction.
 	 * </ul>
 	 * @param[out] txn Address where the new #MDB_txn handle will be stored
 	 * @return A non-zero error value on failure and 0 on success. Some possible
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 03f0a19ea1..45d29f1f14 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1512,7 +1512,7 @@ struct MDB_txn {
  *	@{
  */
 	/** #mdb_txn_begin() flags */
-#define MDB_TXN_BEGIN_FLAGS	(MDB_NOMETASYNC|MDB_NOSYNC|MDB_RDONLY)
+#define MDB_TXN_BEGIN_FLAGS	(MDB_NOMETASYNC|MDB_NOSYNC|MDB_RDONLY|MDB_ROTXN_RESET)
 #define MDB_TXN_NOMETASYNC	MDB_NOMETASYNC	/**< don't sync meta for this txn on commit */
 #define MDB_TXN_NOSYNC		MDB_NOSYNC	/**< don't sync this txn on commit */
 #define MDB_TXN_RDONLY		MDB_RDONLY	/**< read-only transaction */
@@ -3487,6 +3487,9 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 	if (env->me_flags & MDB_RDONLY & ~flags) /* write txn in RDONLY env */
 		return MDB_IS_READONLY;
 
+	if ((flags & MDB_ROTXN_RESET) && !(flags & MDB_RDONLY))	/* MDB_ROTXN_RESET requires MDB_RDONLY */
+		return EINVAL;
+
 	if (parent) {
 		/* Nested transactions:
 		 * Only write txns may have nested txns;
@@ -3606,7 +3609,12 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 	} else { /* MDB_RDONLY */
 		txn->mt_dbiseqs = env->me_dbiseqs;
 renew:
-		rc = mdb_txn_renew0(txn);
+		if (F_ISSET(flags, MDB_ROTXN_RESET)) {
+			rc = MDB_SUCCESS;
+			flags ^= (MDB_ROTXN_RESET|MDB_TXN_FINISHED);
+		} else {
+			rc = mdb_txn_renew0(txn);
+		}
 	}
 	if (rc) {
 		if (txn != env->me_txn0) {

From 7a1f36d51a27a8b638e470551cff99302cfa5048 Mon Sep 17 00:00:00 2001
From: Nir Soffer <nirsof@gmail.com>
Date: Sat, 10 Feb 2018 17:06:09 +0000
Subject: [PATCH 489/504] ITS#8803 LMDB: add tool option to disable locking

For use with environments that use their own locking mechanism
instead of LMDB's internal locking.
---
 libraries/liblmdb/mdb_copy.1 | 6 ++++++
 libraries/liblmdb/mdb_copy.c | 4 +++-
 libraries/liblmdb/mdb_drop.1 | 8 +++++++-
 libraries/liblmdb/mdb_drop.c | 8 ++++++--
 libraries/liblmdb/mdb_dump.1 | 8 +++++++-
 libraries/liblmdb/mdb_dump.c | 8 ++++++--
 libraries/liblmdb/mdb_load.1 | 8 +++++++-
 libraries/liblmdb/mdb_load.c | 8 ++++++--
 libraries/liblmdb/mdb_stat.1 | 8 +++++++-
 libraries/liblmdb/mdb_stat.c | 8 ++++++--
 10 files changed, 61 insertions(+), 13 deletions(-)

diff --git a/libraries/liblmdb/mdb_copy.1 b/libraries/liblmdb/mdb_copy.1
index db0c973a5e..90297959ca 100644
--- a/libraries/liblmdb/mdb_copy.1
+++ b/libraries/liblmdb/mdb_copy.1
@@ -12,6 +12,8 @@ mdb_copy \- LMDB environment copy tool
 [\c
 .BR \-n ]
 [\c
+.BR \-L ]
+[\c
 .BR \-v ]
 [\c
 .BI \-m \ module
@@ -47,6 +49,10 @@ Currently it fails if the environment has suffered a page leak.
 .BR \-n
 Open LDMB environment(s) which do not use subdirectories.
 .TP
+.BR \-L
+Access the LMDB environment without any locking. If concurrent
+access is anticipated, the caller must manage all concurrency itself.
+.TP
 .BR \-v
 Use the previous environment state instead of the latest state.
 This may be useful if the latest state has been corrupted.
diff --git a/libraries/liblmdb/mdb_copy.c b/libraries/liblmdb/mdb_copy.c
index 522d315cf9..34314273b5 100644
--- a/libraries/liblmdb/mdb_copy.c
+++ b/libraries/liblmdb/mdb_copy.c
@@ -41,6 +41,8 @@ int main(int argc,char * argv[])
 	for (; argc > 1 && argv[1][0] == '-'; argc--, argv++) {
 		if (argv[1][1] == 'n' && argv[1][2] == '\0')
 			flags |= MDB_NOSUBDIR;
+		else if (argv[1][1] == 'L' && argv[1][2] == '\0')
+			flags |= MDB_NOLOCK;
 		else if (argv[1][1] == 'v' && argv[1][2] == '\0')
 			flags |= MDB_PREVSNAPSHOT;
 		else if (argv[1][1] == 'c' && argv[1][2] == '\0')
@@ -61,7 +63,7 @@ int main(int argc,char * argv[])
 	}
 
 	if (argc<2 || argc>3) {
-		fprintf(stderr, "usage: %s [-V] [-c] [-n] [-v] [-m module [-w password]] srcpath [dstpath]\n", progname);
+		fprintf(stderr, "usage: %s [-V] [-c] [-n] [-L] [-v] [-m module [-w password]] srcpath [dstpath]\n", progname);
 		exit(EXIT_FAILURE);
 	}
 
diff --git a/libraries/liblmdb/mdb_drop.1 b/libraries/liblmdb/mdb_drop.1
index 36a73e1088..c1b64bb139 100644
--- a/libraries/liblmdb/mdb_drop.1
+++ b/libraries/liblmdb/mdb_drop.1
@@ -10,6 +10,8 @@ mdb_drop \- LMDB database delete tool
 [\c
 .BR \-n ]
 [\c
+.BR \-L ]
+[\c
 .BR \-d ]
 [\c
 .BI \-m \ module
@@ -29,7 +31,11 @@ environment.
 Write the library version number to the standard output, and exit.
 .TP
 .BR \-n
-Operate on an LMDB database which does not use subdirectories.
+Operate on an LMDB environment which does not use subdirectories.
+.TP
+.BR \-L
+Access the LMDB environment without any locking. If concurrent
+access is anticipated, the caller must manage all concurrency itself.
 .TP
 .BR \-d
 Delete the specified database, don't just empty it.
diff --git a/libraries/liblmdb/mdb_drop.c b/libraries/liblmdb/mdb_drop.c
index 2707e7b0f4..93f3d350e4 100644
--- a/libraries/liblmdb/mdb_drop.c
+++ b/libraries/liblmdb/mdb_drop.c
@@ -29,7 +29,7 @@ static void dumpsig( int sig )
 
 static void usage(char *prog)
 {
-	fprintf(stderr, "usage: %s [-V] [-n] [-d] [-m module [-w password]] [-s subdb] dbpath\n", prog);
+	fprintf(stderr, "usage: %s [-V] [-n] [-L] [-d] [-m module [-w password]] [-s subdb] dbpath\n", prog);
 	exit(EXIT_FAILURE);
 }
 
@@ -54,10 +54,11 @@ int main(int argc, char *argv[])
 	/* -d: delete the db, don't just empty it
 	 * -s: drop the named subDB
 	 * -n: use NOSUBDIR flag on env_open
+	 * -L: use NOLOCK flag on env_open
 	 * -V: print version and exit
 	 * (default) empty the main DB
 	 */
-	while ((i = getopt(argc, argv, "dm:ns:w:V")) != EOF) {
+	while ((i = getopt(argc, argv, "dm:nLs:w:V")) != EOF) {
 		switch(i) {
 		case 'V':
 			printf("%s\n", MDB_VERSION_STRING);
@@ -69,6 +70,9 @@ int main(int argc, char *argv[])
 		case 'n':
 			envflags |= MDB_NOSUBDIR;
 			break;
+		case 'L':
+			envflags |= MDB_NOLOCK;
+			break;
 		case 's':
 			subname = optarg;
 			break;
diff --git a/libraries/liblmdb/mdb_dump.1 b/libraries/liblmdb/mdb_dump.1
index 068c037862..e2ac8defda 100644
--- a/libraries/liblmdb/mdb_dump.1
+++ b/libraries/liblmdb/mdb_dump.1
@@ -18,6 +18,8 @@ mdb_dump \- LMDB environment export tool
 [\c
 .BR \-v ]
 [\c
+.BR \-L ]
+[\c
 .BR \-p ]
 [\c
 .BI \-m \ module
@@ -54,12 +56,16 @@ List the databases stored in the environment. Just the
 names will be listed, no data will be output.
 .TP
 .BR \-n
-Dump an LMDB database which does not use subdirectories.
+Dump an LMDB environment which does not use subdirectories.
 .TP
 .BR \-v
 Use the previous environment state instead of the latest state.
 This may be useful if the latest state has been corrupted.
 .TP
+.BR \-L
+Access the LMDB environment without any locking. If concurrent access is
+anticipated, the caller must manage all concurrency itself.
+.TP
 .BR \-p
 If characters in either the key or data items are printing characters (as
 defined by isprint(3)), output them directly. This option permits users to
diff --git a/libraries/liblmdb/mdb_dump.c b/libraries/liblmdb/mdb_dump.c
index 7a79eaec64..a31797f5d8 100644
--- a/libraries/liblmdb/mdb_dump.c
+++ b/libraries/liblmdb/mdb_dump.c
@@ -160,7 +160,7 @@ static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name)
 
 static void usage(char *prog)
 {
-	fprintf(stderr, "usage: %s [-V] [-f output] [-i txnid] [-l] [-n] [-p] [-v] [-m module [-w password]] [-a|-s subdb] dbpath\n", prog);
+	fprintf(stderr, "usage: %s [-V] [-f output] [-i txnid] [-l] [-n] [-L] [-p] [-v] [-m module [-w password]] [-a|-s subdb] dbpath\n", prog);
 	exit(EXIT_FAILURE);
 }
 
@@ -185,6 +185,7 @@ int main(int argc, char *argv[])
 	/* -a: dump all subDBs
 	 * -s: dump only the named subDB
 	 * -n: use NOSUBDIR flag on env_open
+	 * -n: use NOLOCK flag on env_open
 	 * -p: use printable characters
 	 * -f: write to file instead of stdout
 	 * -i: do incremental dump from txnid
@@ -192,7 +193,7 @@ int main(int argc, char *argv[])
 	 * -V: print version and exit
 	 * (default) dump only the main DB
 	 */
-	while ((i = getopt(argc, argv, "af:i:lm:nps:vw:V")) != EOF) {
+	while ((i = getopt(argc, argv, "af:i:lm:nLps:vw:V")) != EOF) {
 		switch(i) {
 		case 'V':
 			printf("%s\n", MDB_VERSION_STRING);
@@ -222,6 +223,9 @@ int main(int argc, char *argv[])
 		case 'v':
 			envflags |= MDB_PREVSNAPSHOT;
 			break;
+		case 'L':
+			envflags |= MDB_NOLOCK;
+			break;
 		case 'p':
 			mode |= PRINT;
 			break;
diff --git a/libraries/liblmdb/mdb_load.1 b/libraries/liblmdb/mdb_load.1
index b99a80397f..c9bec8293c 100644
--- a/libraries/liblmdb/mdb_load.1
+++ b/libraries/liblmdb/mdb_load.1
@@ -20,6 +20,8 @@ mdb_load \- LMDB environment import tool
 [\c
 .BI \-w \ password\fR]]
 [\c
+.BR \-L ]
+[\c
 .BI \-s \ subdb\fR]
 [\c
 .BR \-N ]
@@ -61,7 +63,7 @@ Read from the specified file instead of from the standard input.
 Load an incremental backup.
 .TP
 .BR \-n
-Load an LMDB database which does not use subdirectories.
+Load an LMDB environment which does not use subdirectories.
 .TP
 .BI \-m \ module
 Load the specified dynamic module to utilize cryptographic functions.
@@ -72,6 +74,10 @@ with page-level checksums or encryption.
 Specify the password for an encrypted environment. This is only
 used if a cryptography module has been loaded.
 .TP
+.BR \-L
+Access the LMDB environment without any locking. If concurrent access is
+anticipated, the caller must manage all concurrency itself.
+.TP
 .BR \-s \ subdb
 Load a specific subdatabase. If no database is specified, data is loaded into the main database.
 .TP
diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c
index d3d736a411..838cff36c3 100644
--- a/libraries/liblmdb/mdb_load.c
+++ b/libraries/liblmdb/mdb_load.c
@@ -302,7 +302,7 @@ badend:
 
 static void usage(void)
 {
-	fprintf(stderr, "usage: %s [-V] [-a] [-f input] [-i] [-n] [-m module [-w password]] [-s name] [-N] [-Q] [-T] dbpath\n", prog);
+	fprintf(stderr, "usage: %s [-V] [-a] [-f input] [-i] [-n] [-L] [-m module [-w password]] [-s name] [-N] [-Q] [-T] dbpath\n", prog);
 	exit(EXIT_FAILURE);
 }
 
@@ -336,13 +336,14 @@ int main(int argc, char *argv[])
 	 * -i: load an incremental dump
 	 * -m: dynamically load a module
 	 * -n: use NOSUBDIR flag on env_open
+	 * -L: use NOLOCK flag on env_open
 	 * -s: load into named subDB
 	 * -N: use NOOVERWRITE on puts
 	 * -Q: quick mode using NOSYNC
 	 * -T: read plaintext
 	 * -V: print version and exit
 	 */
-	while ((i = getopt(argc, argv, "af:im:ns:w:NQTV")) != EOF) {
+	while ((i = getopt(argc, argv, "af:im:nLs:w:NQTV")) != EOF) {
 		switch(i) {
 		case 'V':
 			printf("%s\n", MDB_VERSION_STRING);
@@ -364,6 +365,9 @@ int main(int argc, char *argv[])
 		case 'n':
 			envflags |= MDB_NOSUBDIR;
 			break;
+		case 'L':
+			envflags |= MDB_NOLOCK;
+			break;
 		case 's':
 			subname = strdup(optarg);
 			break;
diff --git a/libraries/liblmdb/mdb_stat.1 b/libraries/liblmdb/mdb_stat.1
index 367bd6f36a..2454df19da 100644
--- a/libraries/liblmdb/mdb_stat.1
+++ b/libraries/liblmdb/mdb_stat.1
@@ -20,6 +20,8 @@ mdb_stat \- LMDB environment status tool
 [\c
 .BI \-w \ password\fR]]
 [\c
+.BR \-L ]
+[\c
 .BR \-r [ r ]]
 [\c
 .BR \-a \ |
@@ -43,7 +45,7 @@ If \fB\-ff\fP is given, summarize each freelist entry.
 If \fB\-fff\fP is given, display the full list of page IDs in the freelist.
 .TP
 .BR \-n
-Display the status of an LMDB database which does not use subdirectories.
+Display the status of an LMDB environment which does not use subdirectories.
 .TP
 .BR \-v
 Use the previous environment state instead of the latest state.
@@ -58,6 +60,10 @@ with page-level checksums or encryption.
 Specify the password for an encrypted environment. This is only
 used if a cryptography module has been loaded.
 .TP
+.BR \-L
+Access the LMDB environment without any locking. If concurrent
+access is anticipated, the caller must manage all concurrency itself.
+.TP
 .BR \-r
 Display information about the environment reader table.
 Shows the process ID, thread ID, and transaction ID for each active
diff --git a/libraries/liblmdb/mdb_stat.c b/libraries/liblmdb/mdb_stat.c
index 9ae31861b8..97f786f869 100644
--- a/libraries/liblmdb/mdb_stat.c
+++ b/libraries/liblmdb/mdb_stat.c
@@ -32,7 +32,7 @@ static void prstat(MDB_stat *ms)
 
 static void usage(char *prog)
 {
-	fprintf(stderr, "usage: %s [-V] [-n] [-e] [-r[r]] [-f[f[f]]] [-v] [-m module [-w password]] [-a|-s subdb] dbpath\n", prog);
+	fprintf(stderr, "usage: %s [-V] [-n] [-L] [-e] [-r[r]] [-f[f[f]]] [-v] [-m module [-w password]] [-a|-s subdb] dbpath\n", prog);
 	exit(EXIT_FAILURE);
 }
 
@@ -61,11 +61,12 @@ int main(int argc, char *argv[])
 	 * -f: print freelist info
 	 * -r: print reader info
 	 * -n: use NOSUBDIR flag on env_open
+	 * -L: use NOLOCK flag on env_open
 	 * -v: use previous snapshot
 	 * -V: print version and exit
 	 * (default) print stat of only the main DB
 	 */
-	while ((i = getopt(argc, argv, "Vaefm:nrs:vw:")) != EOF) {
+	while ((i = getopt(argc, argv, "Vaefm:nLrs:vw:")) != EOF) {
 		switch(i) {
 		case 'V':
 			printf("%s\n", MDB_VERSION_STRING);
@@ -88,6 +89,9 @@ int main(int argc, char *argv[])
 		case 'v':
 			envflags |= MDB_PREVSNAPSHOT;
 			break;
+		case 'L':
+			envflags |= MDB_NOLOCK;
+			break;
 		case 'r':
 			rdrinfo++;
 			break;

From ed917c6c02468b7a1462049eecb709b9770dd3bb Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Sun, 26 Apr 2026 15:07:40 +0100
Subject: [PATCH 490/504] Revert "ITS#8250 LMDB: add option MDB_ROTXN_RESET
 flag for txn_begin"

This reverts commit 4cd8e451679511c8fbeff68f8a97cbe7b1268b48.
Cost/benefit makes no sense.
---
 libraries/liblmdb/lmdb.h |  9 ---------
 libraries/liblmdb/mdb.c  | 12 ++----------
 2 files changed, 2 insertions(+), 19 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 3969ee81f7..da7b3c94a6 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -372,13 +372,6 @@ typedef void (MDB_sum_func)(const MDB_val *src, MDB_val *dst, const MDB_val *key
 #define MDB_REMAP_CHUNKS	0x4000000
 /** @} */
 
-/** @defgroup	mdb_txn_begin	Transaction Flags
- *	@{
- */
-	/** don't initialize read-only txn; #mdb_txn_renew() must be called before first use */
-#define MDB_ROTXN_RESET	0x8000000
-/** @} */
-
 /**	@defgroup	mdb_dbi_open	Database Flags
  *	@{
  */
@@ -1150,8 +1143,6 @@ int mdb_env_set_checksum(MDB_env *env, MDB_sum_func *func, unsigned int size);
 	 *		Don't flush system buffers to disk when committing this transaction.
 	 *	<li>#MDB_NOMETASYNC
 	 *		Flush system buffers but omit metadata flush when committing this transaction.
-	 *	<li>#MDB_ROTXN_RESET
-	 *		Must call #mdb_txn_renew() before using this readonly transaction.
 	 * </ul>
 	 * @param[out] txn Address where the new #MDB_txn handle will be stored
 	 * @return A non-zero error value on failure and 0 on success. Some possible
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 45d29f1f14..03f0a19ea1 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -1512,7 +1512,7 @@ struct MDB_txn {
  *	@{
  */
 	/** #mdb_txn_begin() flags */
-#define MDB_TXN_BEGIN_FLAGS	(MDB_NOMETASYNC|MDB_NOSYNC|MDB_RDONLY|MDB_ROTXN_RESET)
+#define MDB_TXN_BEGIN_FLAGS	(MDB_NOMETASYNC|MDB_NOSYNC|MDB_RDONLY)
 #define MDB_TXN_NOMETASYNC	MDB_NOMETASYNC	/**< don't sync meta for this txn on commit */
 #define MDB_TXN_NOSYNC		MDB_NOSYNC	/**< don't sync this txn on commit */
 #define MDB_TXN_RDONLY		MDB_RDONLY	/**< read-only transaction */
@@ -3487,9 +3487,6 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 	if (env->me_flags & MDB_RDONLY & ~flags) /* write txn in RDONLY env */
 		return MDB_IS_READONLY;
 
-	if ((flags & MDB_ROTXN_RESET) && !(flags & MDB_RDONLY))	/* MDB_ROTXN_RESET requires MDB_RDONLY */
-		return EINVAL;
-
 	if (parent) {
 		/* Nested transactions:
 		 * Only write txns may have nested txns;
@@ -3609,12 +3606,7 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 	} else { /* MDB_RDONLY */
 		txn->mt_dbiseqs = env->me_dbiseqs;
 renew:
-		if (F_ISSET(flags, MDB_ROTXN_RESET)) {
-			rc = MDB_SUCCESS;
-			flags ^= (MDB_ROTXN_RESET|MDB_TXN_FINISHED);
-		} else {
-			rc = mdb_txn_renew0(txn);
-		}
+		rc = mdb_txn_renew0(txn);
 	}
 	if (rc) {
 		if (txn != env->me_txn0) {

From 134e5ace0409ab04d7ad38851446c1b996f64908 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 27 Apr 2026 14:59:00 +0100
Subject: [PATCH 491/504] ITS#7772 LMDB: fix sub-page growth

Check for available space before deciding to grow the sub-page.
The test program will still leave some unused space, but after the
first iteration the extra space will be used instead of growing more.
---
 libraries/liblmdb/mdb.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 03f0a19ea1..73ebc910e0 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -8707,8 +8707,15 @@ more:
 				switch (flags) {
 				default:
 					if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) {
+						unsigned int left = SIZELEFT(fp);
 						offset = EVEN(NODESIZE + sizeof(indx_t) +
 							data->mv_size);
+						/* if there's enough space, just use it */
+						if (offset < left)
+							offset = 0;
+						else
+						/* else grow by whatever we need */
+							offset -= left;
 						break;
 					}
 					offset = fp->mp_pad;

From 4e8a093a861dbfd4adadd14fb66af9d233bf297e Mon Sep 17 00:00:00 2001
From: Kerollmops <clement@meilisearch.com>
Date: Mon, 27 Apr 2026 13:24:44 +0200
Subject: [PATCH 492/504] ITS#10454 LMDB: fix prev commit for Mac OSX +
 WRITEMAP

---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 73ebc910e0..75bcaad837 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -4885,7 +4885,7 @@ mdb_env_write_meta(MDB_txn *txn)
 				goto fail;
 			}
 #if defined(__APPLE__)
-			if (MDB_FDATASYNC(env->me_mfd)) {
+			if (MDB_FDATASYNC(env->me_fd)) {
 				rc = ErrCode();
 				goto fail;
 			}

From 25f9efab91340b2f8e232e05baba9007c1901c9c Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 27 Apr 2026 17:28:19 +0100
Subject: [PATCH 493/504] ITS#9388 LMDB: fix mdb_stat page counts for dupsort
 DBs

---
 libraries/liblmdb/mdb.c | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 75bcaad837..ba7ce9df94 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -8464,6 +8464,18 @@ mdb_cursor_touch(MDB_cursor *mc)
 	return rc;
 }
 
+static void
+mdb_subdb_adjust(MDB_cursor *mc, MDB_db *old, MDB_db *new)
+{
+	int delta;
+
+	delta = new->md_branch_pages - old->md_branch_pages;
+	mc->mc_db->md_branch_pages += delta;
+
+	delta = new->md_leaf_pages - old->md_leaf_pages;
+	mc->mc_db->md_leaf_pages += delta;
+}
+
 /** Do not spill pages to disk if txn is getting full, may fail instead */
 #define MDB_NOSPILL	0x8000
 
@@ -8762,6 +8774,7 @@ prep_subDB:
 					flags |= F_DUPDATA|F_SUBDATA;
 					dummy.md_root = mp->mp_pgno;
 					sub_root = mp;
+					mc->mc_db->md_leaf_pages++;
 			}
 			if (mp != fp) {
 				MP_FLAGS(mp) = fp_flags;
@@ -8961,6 +8974,7 @@ put_sub:
 			rc = _mdb_cursor_put(&mc->mc_xcursor->mx_cursor, data, &xdata, xflags);
 			if (flags & F_SUBDATA) {
 				void *db = NODEDATA(leaf);
+				mdb_subdb_adjust(mc, db, &mc->mc_xcursor->mx_db);
 				memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDB_db));
 			}
 			insert_data = mc->mc_xcursor->mx_db.md_entries - ecount;
@@ -9058,6 +9072,7 @@ _mdb_cursor_del(MDB_cursor *mc, unsigned int flags)
 				if (leaf->mn_flags & F_SUBDATA) {
 					/* update subDB info */
 					void *db = NODEDATA(leaf);
+					mdb_subdb_adjust(mc, db, &mc->mc_xcursor->mx_db);
 					memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDB_db));
 				} else {
 					MDB_cursor *m2;

From 0ff5b99e40cf4e87f00ad562a4a1c97faae48a4a Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 27 Apr 2026 18:37:15 +0100
Subject: [PATCH 494/504] ITS#8335 LMDB: reject MDB_MULTIPLE put with 0 items

Also check for overflow of count and size.
---
 libraries/liblmdb/mdb.c | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index ba7ce9df94..b665a62002 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -8505,10 +8505,19 @@ _mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data,
 	 * early failures.
 	 */
 	if (flags & MDB_MULTIPLE) {
+		size_t tmp;
+		if (!data[1].mv_size)
+			return EINVAL;
+
 		dcount = data[1].mv_size;
 		data[1].mv_size = 0;
 		if (!F_ISSET(mc->mc_db->md_flags, MDB_DUPFIXED))
 			return MDB_INCOMPATIBLE;
+
+		/* check for overflow */
+		tmp = data[0].mv_size * dcount;
+		if (tmp/dcount != data[0].mv_size)
+			return MDB_BAD_VALSIZE;
 	}
 
 	nospill = flags & MDB_NOSPILL;

From fb4737e91c429ab532c01539e3f327f013c384e3 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 27 Apr 2026 19:31:44 +0100
Subject: [PATCH 495/504] ITS#10203 LMDB: add lmdb.pc to Makefile

---
 libraries/liblmdb/Makefile | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile
index badb1165d2..6d486fad7b 100644
--- a/libraries/liblmdb/Makefile
+++ b/libraries/liblmdb/Makefile
@@ -27,12 +27,15 @@ CFLAGS	= $(THREADS) $(OPT) $(W) $(XCFLAGS)
 LDFLAGS = $(THREADS)
 LDLIBS	= 
 SOLIBS	= 
+SOEXT	= .so
+LDL		= -ldl
+
 LIBVER	= 1
 ABIVER	= 0
 VEREXT	= $(LIBVER).$(ABIVER)
-SOEXT	= .so
+LMDB_VERSION	= 1.0.0
 SOFULL	= $(SOEXT).$(VEREXT)
-LDL		= -ldl
+
 prefix	= /usr/local
 exec_prefix = $(prefix)
 bindir = $(exec_prefix)/bin
@@ -55,7 +58,7 @@ VERSION_OPT	= -Wl,-soname,$(SOVER)
 # For MacOSX:
 #VERSION_OPT	= -Wl,-current_version,$(VEREXT)
 
-all:	$(ILIBS) $(ILIBS2) $(PROGS)
+all:	$(ILIBS) $(ILIBS2) $(PROGS) lmdb.pc
 # Requires CPPFLAGS=-DMDB_VL32 and/or -DMDB_RPAGE_CACHE
 rall:	all $(RPROGS)
 
@@ -137,6 +140,19 @@ module.lo: module.c lmdb.h
 %.o:	%.c lmdb.h
 	$(CC) $(CFLAGS) $(CPPFLAGS) -c $<
 
+lmdb.pc: Makefile
+	@echo "prefix=$(prefix)" > $@
+	@echo "exec_prefix=$(exec_prefix)" >>$@
+	@echo "includedir=$(includedir)" >>$@
+	@echo "libdir=$(libdir)" >>$@
+	@echo >>$@
+	@echo "Name: lmdb (OpenLDAP)" >>$@
+	@echo "Description: OpenLDAP Lightning Memory Mapped Database library" >>$@
+	@echo "URL: https://www.openldap.org" >>$@
+	@echo "Version: $(LMDB_VERSION)" >>$@
+	@echo "Cflags: $(THREADS) $(XCFLAGS)" >>$@
+	@echo "Libs: $(LDL)" >>$@
+
 COV_FLAGS=-fprofile-arcs -ftest-coverage
 COV_OBJS=xmdb.o xmidl.o
 

From 340f04593e3a05db615b3444fac9f3572ed75e3f Mon Sep 17 00:00:00 2001
From: Markus Junginger <markus@objectbox.io>
Date: Tue, 28 Jul 2020 20:04:50 +0200
Subject: [PATCH 496/504] ITS#9291 LMDB: quick sanity check on env open

Validate root page numbers and page sum against file size while opening env.
---
 libraries/liblmdb/mdb.c | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index b665a62002..5a06d7be67 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -5528,6 +5528,31 @@ mdb_env_open2(MDB_env *env, int prev)
 		if (env->me_mapsize == DEFAULT_MAPSIZE)
 			env->me_mapsize = 0;
 	}
+
+	/* Verify that root pages are actually inside the file */
+	{
+		mdb_size_t fsize = 0;
+		if (mdb_fsize(env->me_fd, &fsize) == MDB_SUCCESS && fsize) {
+			MDB_dbi dbi;
+			pgno_t maxpgno = fsize / env->me_psize;
+			pgno_t sum = NUM_METAS;
+			for (dbi = FREE_DBI; dbi <= MAIN_DBI; dbi++) {
+				MDB_db *db = &meta.mm_dbs[dbi];
+				if (db->md_root != P_INVALID) {
+					if (db->md_root > maxpgno)
+						return MDB_PAGE_NOTFOUND;
+					sum += db->md_branch_pages;
+					sum += db->md_leaf_pages;
+					sum += db->md_overflow_pages;
+				}
+			}
+			/* Slack sanity check as sum ignores pages from 2nd meta. */
+			/* (2nd meta may be corrupted; thus do not rely on it.)  */
+			if (sum > maxpgno)
+				return MDB_CORRUPTED;
+		}
+	}
+
 	/* Was a mapsize configured? */
 	if (!env->me_mapsize) {
 		env->me_mapsize = meta.mm_mapsize;

From 597e77a1f40211c6d5fd74bd6cb13a65eb6fac65 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Tue, 28 Apr 2026 17:09:52 +0100
Subject: [PATCH 497/504] Creating LMDB 1.0 release branch

---
 libraries/liblmdb/lmdb.h | 10 +++++-----
 libraries/liblmdb/mdb.c  |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index da7b3c94a6..7d0d7e1023 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -136,7 +136,7 @@
  *
  *	@author	Howard Chu, Symas Corporation.
  *
- *	@copyright Copyright 2011-2021 Howard Chu, Symas Corp. All rights reserved.
+ *	@copyright Copyright 2011-2026 Howard Chu, Symas Corp. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted only as authorized by the OpenLDAP
@@ -230,11 +230,11 @@ typedef int mdb_filehandle_t;
  *	@{
  */
 /** Library major version */
-#define MDB_VERSION_MAJOR	0
+#define MDB_VERSION_MAJOR	1
 /** Library minor version */
-#define MDB_VERSION_MINOR	9
+#define MDB_VERSION_MINOR	0
 /** Library patch version */
-#define MDB_VERSION_PATCH	90
+#define MDB_VERSION_PATCH	0
 
 /** Combine args a,b,c into a single integer for easy version comparisons */
 #define MDB_VERINT(a,b,c)	(((a) << 24) | ((b) << 16) | (c))
@@ -244,7 +244,7 @@ typedef int mdb_filehandle_t;
 	MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH)
 
 /** The release date of this library version */
-#define MDB_VERSION_DATE	"May 1, 2017"
+#define MDB_VERSION_DATE	"April 28, 2026"
 
 /** A stringifier for the version info */
 #define MDB_VERSTR(a,b,c,d)	"LMDB " #a "." #b "." #c ": (" d ")"
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 5a06d7be67..3f7ce4b4a0 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -707,7 +707,7 @@ static txnid_t mdb_debug_start;
 	 *	#MDB_DUPSORT data items must fit on a node in a regular page.
 	 */
 #ifndef MDB_MAXKEYSIZE
-#define MDB_MAXKEYSIZE	 ((MDB_DEVEL) ? 0 : 511)
+#define MDB_MAXKEYSIZE	0
 #endif
 
 	/**	The maximum size of a key we can write to the environment. */

From f8ec414d7dad8fe5f235c4df2ddf98e111108adc Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 29 Apr 2026 18:28:57 +0100
Subject: [PATCH 498/504] ITS#9619 LMDB: more for prev commit

Also needed to set overflow pgno.
---
 libraries/liblmdb/mdb.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 3f7ce4b4a0..80654cd672 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -11286,8 +11286,9 @@ mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags)
 						}
 						mo = (MDB_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]);
 						memcpy(mo, omp, my->mc_env->me_psize);
-						mo->mp_txnid = 1;
 						ovp.op_pgno = my->mc_next_pgno;
+						mo->mp_pgno = ovp.op_pgno;
+						mo->mp_txnid = 1;
 						ovp.op_txnid = 1;
 						memcpy(NODEDATA(ni), &ovp, sizeof(ovp));
 						my->mc_next_pgno += ovp.op_pages;

From 11ebc033d258da3b3f76b1be77bac750768ef2ae Mon Sep 17 00:00:00 2001
From: Christopher Zimmermann <madroach@gmerlin.de>
Date: Sat, 20 Apr 2019 23:06:51 +0200
Subject: [PATCH 499/504] ITS#9011 LMDB: add mdb_txn_flags()

---
 libraries/liblmdb/lmdb.h | 8 ++++++++
 libraries/liblmdb/mdb.c  | 7 +++++++
 2 files changed, 15 insertions(+)

diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h
index 7d0d7e1023..2653e5885f 100644
--- a/libraries/liblmdb/lmdb.h
+++ b/libraries/liblmdb/lmdb.h
@@ -1177,6 +1177,14 @@ MDB_env *mdb_txn_env(MDB_txn *txn);
 	 */
 mdb_size_t mdb_txn_id(MDB_txn *txn);
 
+	/** @brief Retrieve the transaction's flags
+	 *
+	 * @param[in] txn A transaction handle returned by #mdb_txn_begin()
+	 * @param[out] flags Address where the flags will be returned.
+	 * @return A non-zero error value on failure and 0 on success.
+	 */
+int mdb_txn_flags(MDB_txn *txn, unsigned int *flags);
+
 	/** @brief Commit all the operations of a transaction into the database.
 	 *
 	 * The transaction handle is freed. It and its cursors must not be used
diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 80654cd672..3424f15039 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -3642,6 +3642,13 @@ mdb_txn_id(MDB_txn *txn)
     return txn->mt_txnid;
 }
 
+int mdb_txn_flags(MDB_txn *txn, unsigned int *flags)
+{
+    if(!txn) return EINVAL;
+    *flags = txn->mt_flags & MDB_RDONLY;
+    return MDB_SUCCESS;
+}
+
 /** Export or close DBI handles opened in this txn. */
 static void
 mdb_dbis_update(MDB_txn *txn, int keep)

From 1924ae9ba85be626824cdfe83bc81effef2422a7 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 4 May 2026 14:47:50 +0100
Subject: [PATCH 500/504] Whitespace cleanup

---
 libraries/liblmdb/mdb.c | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index 3424f15039..f786c64366 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -3638,15 +3638,15 @@ mdb_txn_env(MDB_txn *txn)
 mdb_size_t
 mdb_txn_id(MDB_txn *txn)
 {
-    if(!txn) return 0;
-    return txn->mt_txnid;
+	if(!txn) return 0;
+	return txn->mt_txnid;
 }
 
 int mdb_txn_flags(MDB_txn *txn, unsigned int *flags)
 {
-    if(!txn) return EINVAL;
-    *flags = txn->mt_flags & MDB_RDONLY;
-    return MDB_SUCCESS;
+	if(!txn) return EINVAL;
+	flags = txn->mt_flags & MDB_RDONLY;
+	return MDB_SUCCESS;
 }
 
 /** Export or close DBI handles opened in this txn. */
@@ -4330,9 +4330,9 @@ bad_write:
 	}
 #endif	/* _WIN32 */
 
-    if (!(env->me_flags & MDB_WRITEMAP)) {
-        /* Don't free pages when using writemap (can only get here in NOSYNC mode in Windows)
-         */
+	if (!(env->me_flags & MDB_WRITEMAP)) {
+		/* Don't free pages when using writemap (can only get here in NOSYNC mode in Windows)
+		 */
 		for (i = keep; ++i <= pagecount; ) {
 			dp = dl[i].mptr;
 			/* This is a page we skipped above */
@@ -7726,7 +7726,7 @@ mdb_node_read(MDB_cursor *mc, MDB_node *leaf, MDB_val *data)
 
 int
 mdb_get(MDB_txn *txn, MDB_dbi dbi,
-    MDB_val *key, MDB_val *data)
+	MDB_val *key, MDB_val *data)
 {
 	MDB_cursor	mc;
 	MDB_xcursor	mx;
@@ -7991,7 +7991,7 @@ mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
 /** Set the cursor on a specific data item. */
 static int
 mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data,
-    MDB_cursor_op op, int *exactp)
+	MDB_cursor_op op, int *exactp)
 {
 	int		 rc;
 	MDB_page	*mp;
@@ -8280,7 +8280,7 @@ mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data)
 
 int
 mdb_cursor_get(MDB_cursor *mc, MDB_val *key, MDB_val *data,
-    MDB_cursor_op op)
+	MDB_cursor_op op)
 {
 	int		 rc;
 	int		 exact = 0;
@@ -8513,7 +8513,7 @@ mdb_subdb_adjust(MDB_cursor *mc, MDB_db *old, MDB_db *new)
 
 static int
 _mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data,
-    unsigned int flags)
+	unsigned int flags)
 {
 	MDB_env		*env;
 	MDB_node	*leaf = NULL;
@@ -9055,7 +9055,7 @@ bad_sub:
 
 int
 mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data,
-    unsigned int flags)
+	unsigned int flags)
 {
 	DKBUF;
 	DDBUF;
@@ -9282,7 +9282,7 @@ mdb_branch_size(MDB_env *env, MDB_val *key)
  */
 static int
 mdb_node_add(MDB_cursor *mc, indx_t indx,
-    MDB_val *key, MDB_val *data, pgno_t pgno, unsigned int flags)
+	MDB_val *key, MDB_val *data, pgno_t pgno, unsigned int flags)
 {
 	unsigned int	 i;
 	size_t		 node_size = NODESIZE;
@@ -10564,7 +10564,7 @@ fail:
 
 int
 mdb_del(MDB_txn *txn, MDB_dbi dbi,
-    MDB_val *key, MDB_val *data)
+	MDB_val *key, MDB_val *data)
 {
 	DKBUF;
 	DDBUF;
@@ -11061,7 +11061,7 @@ done:
 
 int
 mdb_put(MDB_txn *txn, MDB_dbi dbi,
-    MDB_val *key, MDB_val *data, unsigned int flags)
+	MDB_val *key, MDB_val *data, unsigned int flags)
 {
 	MDB_cursor mc;
 	MDB_xcursor mx;

From c2b1cab5dc190fef836b37512abbfd80c0e8b331 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Mon, 4 May 2026 14:50:25 +0100
Subject: [PATCH 501/504] ITS#9011 LMDB: fix typo in prev commit

---
 libraries/liblmdb/mdb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
index f786c64366..a701f8dd6e 100644
--- a/libraries/liblmdb/mdb.c
+++ b/libraries/liblmdb/mdb.c
@@ -3645,7 +3645,7 @@ mdb_txn_id(MDB_txn *txn)
 int mdb_txn_flags(MDB_txn *txn, unsigned int *flags)
 {
 	if(!txn) return EINVAL;
-	flags = txn->mt_flags & MDB_RDONLY;
+	*flags = txn->mt_flags & MDB_RDONLY;
 	return MDB_SUCCESS;
 }
 

From 6966f369f106e7114dd8db81c92a7e0be2f9bc43 Mon Sep 17 00:00:00 2001
From: Quanah Gibson-Mount <quanah@openldap.org>
Date: Wed, 6 May 2026 14:43:25 +0000
Subject: [PATCH 502/504] Add changes file for LMDB 1.0 series

---
 libraries/liblmdb/CHANGES | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 libraries/liblmdb/CHANGES

diff --git a/libraries/liblmdb/CHANGES b/libraries/liblmdb/CHANGES
new file mode 100644
index 0000000000..6b59127e8f
--- /dev/null
+++ b/libraries/liblmdb/CHANGES
@@ -0,0 +1,3 @@
+LMDB 1.0 Change Log
+
+LMDB 1.0.0 Release (2026/04/28)

From f3ef35bbfbaaab8984b1a4eeb553643fb507862f Mon Sep 17 00:00:00 2001
From: Josh Soref <jsoref@users.noreply.github.com>
Date: Sun, 26 Feb 2017 07:49:31 +0000
Subject: [PATCH 503/504] ITS#8605 - spelling fixes

---
 libraries/liblmdb/Doxyfile | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/libraries/liblmdb/Doxyfile b/libraries/liblmdb/Doxyfile
index 5ca2cfe8f6..e51fe756a0 100644
--- a/libraries/liblmdb/Doxyfile
+++ b/libraries/liblmdb/Doxyfile
@@ -57,7 +57,7 @@ CREATE_SUBDIRS         = NO
 # Croatian, Czech, Danish, Dutch, Esperanto, Farsi, Finnish, French, German,
 # Greek, Hungarian, Italian, Japanese, Japanese-en (Japanese with English
 # messages), Korean, Korean-en, Lithuanian, Norwegian, Macedonian, Persian,
-# Polish, Portuguese, Romanian, Russian, Serbian, Serbian-Cyrilic, Slovak,
+# Polish, Portuguese, Romanian, Russian, Serbian, Serbian-Cyrillic, Slovak,
 # Slovene, Spanish, Swedish, Ukrainian, and Vietnamese.
 
 OUTPUT_LANGUAGE        = English
@@ -211,7 +211,7 @@ OPTIMIZE_OUTPUT_VHDL   = NO
 # parses. With this tag you can assign which parser to use for a given extension.
 # Doxygen has a built-in mapping, but you can override or extend it using this
 # tag. The format is ext=language, where ext is a file extension, and language
-# is one of the parsers supported by doxygen: IDL, Java, Javascript, CSharp, C,
+# is one of the parsers supported by doxygen: IDL, Java, JavaScript, CSharp, C,
 # C++, D, PHP, Objective-C, Python, Fortran, VHDL, C, C++. For instance to make
 # doxygen treat .inc files as Fortran files (default is PHP), and .f files as C
 # (default is Fortran), use: inc=Fortran f=C. Note that for custom extensions
@@ -550,7 +550,7 @@ WARN_IF_UNDOCUMENTED   = YES
 
 WARN_IF_DOC_ERROR      = YES
 
-# This WARN_NO_PARAMDOC option can be abled to get warnings for
+# This WARN_NO_PARAMDOC option can be enabled to get warnings for
 # functions that are documented, but have no documentation for their parameters
 # or return value. If set to NO (the default) doxygen will only warn about
 # wrong or incomplete parameter documentation, but not about the absence of
@@ -1051,7 +1051,7 @@ EXT_LINKS_IN_WINDOW    = NO
 
 FORMULA_FONTSIZE       = 10
 
-# Use the FORMULA_TRANPARENT tag to determine whether or not the images
+# Use the FORMULA_TRANSPARENT tag to determine whether or not the images
 # generated for formulas are transparent PNGs. Transparent PNGs are
 # not supported properly for IE 6.0, but are supported on all modern browsers.
 # Note that when changing this option you need to delete any form_*.png files
@@ -1071,7 +1071,7 @@ SEARCHENGINE           = YES
 
 # When the SERVER_BASED_SEARCH tag is enabled the search engine will be
 # implemented using a PHP enabled web server instead of at the web client
-# using Javascript. Doxygen will generate the search PHP script and index
+# using JavaScript. Doxygen will generate the search PHP script and index
 # file to put on the web server. The advantage of the server
 # based approach is that it scales better to large projects and allows
 # full text search. The disadvances is that it is more difficult to setup

From f57fb27496b99bb9b4ee2282d98b34363958328c Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@openldap.org>
Date: Wed, 6 May 2026 21:23:40 +0100
Subject: [PATCH 504/504] ITS#9009 LMDB: update some version and copyright
 stamps

---
 libraries/liblmdb/mdb_copy.1 | 4 ++--
 libraries/liblmdb/mdb_drop.1 | 4 ++--
 libraries/liblmdb/mdb_dump.1 | 4 ++--
 libraries/liblmdb/mdb_load.1 | 4 ++--
 libraries/liblmdb/mdb_stat.1 | 4 ++--
 libraries/liblmdb/midl.c     | 4 ++--
 libraries/liblmdb/midl.h     | 4 ++--
 7 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/libraries/liblmdb/mdb_copy.1 b/libraries/liblmdb/mdb_copy.1
index 90297959ca..3f1cdf9159 100644
--- a/libraries/liblmdb/mdb_copy.1
+++ b/libraries/liblmdb/mdb_copy.1
@@ -1,5 +1,5 @@
-.TH MDB_COPY 1 "2017/07/31" "LMDB 0.9.90"
-.\" Copyright 2012-2021 Howard Chu, Symas Corp. All Rights Reserved.
+.TH MDB_COPY 1 "2026/04/28" "LMDB 1.0.0"
+.\" Copyright 2012-2026 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_copy \- LMDB environment copy tool
diff --git a/libraries/liblmdb/mdb_drop.1 b/libraries/liblmdb/mdb_drop.1
index c1b64bb139..ec95ca2ff7 100644
--- a/libraries/liblmdb/mdb_drop.1
+++ b/libraries/liblmdb/mdb_drop.1
@@ -1,5 +1,5 @@
-.TH MDB_DROP 1 "2017/11/19" "LMDB 0.9.90"
-.\" Copyright 2014-2021 Howard Chu, Symas Corp. All Rights Reserved.
+.TH MDB_DROP 1 "2026/04/28" "LMDB 1.0.0"
+.\" Copyright 2014-2026 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_drop \- LMDB database delete tool
diff --git a/libraries/liblmdb/mdb_dump.1 b/libraries/liblmdb/mdb_dump.1
index e2ac8defda..858db1d47e 100644
--- a/libraries/liblmdb/mdb_dump.1
+++ b/libraries/liblmdb/mdb_dump.1
@@ -1,5 +1,5 @@
-.TH MDB_DUMP 1 "2017/07/31" "LMDB 0.9.90"
-.\" Copyright 2014-2021 Howard Chu, Symas Corp. All Rights Reserved.
+.TH MDB_DUMP 1 "2026/04/28" "LMDB 1.0.0"
+.\" Copyright 2014-2026 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_dump \- LMDB environment export tool
diff --git a/libraries/liblmdb/mdb_load.1 b/libraries/liblmdb/mdb_load.1
index c9bec8293c..367b36d89a 100644
--- a/libraries/liblmdb/mdb_load.1
+++ b/libraries/liblmdb/mdb_load.1
@@ -1,5 +1,5 @@
-.TH MDB_LOAD 1 "2015/09/30" "LMDB 0.9.90"
-.\" Copyright 2014-2021 Howard Chu, Symas Corp. All Rights Reserved.
+.TH MDB_LOAD 1 "2026/04/28" "LMDB 1.0.0"
+.\" Copyright 2014-2026 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_load \- LMDB environment import tool
diff --git a/libraries/liblmdb/mdb_stat.1 b/libraries/liblmdb/mdb_stat.1
index 2454df19da..f7d9fe5aab 100644
--- a/libraries/liblmdb/mdb_stat.1
+++ b/libraries/liblmdb/mdb_stat.1
@@ -1,5 +1,5 @@
-.TH MDB_STAT 1 "2017/07/31" "LMDB 0.9.90"
-.\" Copyright 2012-2021 Howard Chu, Symas Corp. All Rights Reserved.
+.TH MDB_STAT 1 "2026/04/28" "LMDB 1.0.0"
+.\" Copyright 2012-2026 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_stat \- LMDB environment status tool
diff --git a/libraries/liblmdb/midl.c b/libraries/liblmdb/midl.c
index 272e557e3f..8a8fda7ab5 100644
--- a/libraries/liblmdb/midl.c
+++ b/libraries/liblmdb/midl.c
@@ -3,8 +3,8 @@
 /* $OpenLDAP$ */
 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
  *
- * Copyright 2000-2021 The OpenLDAP Foundation.
- * Portions Copyright 2001-2021 Howard Chu, Symas Corp.
+ * Copyright 2000-2026 The OpenLDAP Foundation.
+ * Portions Copyright 2001-2026 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/libraries/liblmdb/midl.h b/libraries/liblmdb/midl.h
index 765708cf6b..1cccb92eab 100644
--- a/libraries/liblmdb/midl.h
+++ b/libraries/liblmdb/midl.h
@@ -11,8 +11,8 @@
 /* $OpenLDAP$ */
 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
  *
- * Copyright 2000-2021 The OpenLDAP Foundation.
- * Portions Copyright 2001-2021 Howard Chu, Symas Corp.
+ * Copyright 2000-2026 The OpenLDAP Foundation.
+ * Portions Copyright 2001-2026 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without