From fc137e39a311818fb636e9db59967406f02edda6 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Mon, 15 Jul 2013 10:57:13 -0700 Subject: [PATCH 1/6] Add mdb_dbi_flags() Retrieve the flags from a DB handle. --- libraries/liblmdb/lmdb.h | 9 +++++++++ libraries/liblmdb/mdb.c | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h index 7c5a63f0e9..c38c7dbf8f 100644 --- a/libraries/liblmdb/lmdb.h +++ b/libraries/liblmdb/lmdb.h @@ -889,6 +889,15 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *d */ int mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *stat); + /** @brief Retrieve the DB flags for a database handle. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[out] flags Address where the flags will be returned. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_dbi_flags(MDB_env *env, MDB_dbi dbi, unsigned int *flags); + /** @brief Close a database handle. * * This call is not mutex protected. Handles should only be closed by diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 2f997737f9..f895c0fb5e 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -7743,6 +7743,15 @@ void mdb_dbi_close(MDB_env *env, MDB_dbi dbi) free(ptr); } +int mdb_dbi_flags(MDB_env *env, MDB_dbi dbi, unsigned int *flags) +{ + /* We could return the flags for the FREE_DBI too but what's the point? */ + if (dbi <= MAIN_DBI || dbi >= env->me_numdbs) + return EINVAL; + *flags = env->me_dbflags[dbi]; + return MDB_SUCCESS; +} + /** Add all the DB's pages to the free list. * @param[in] mc Cursor on the DB to free. * @param[in] subs non-Zero to check for sub-DBs in this DB. From 49289f0d2e8b5b2fb0295606938c2a4007800e3b Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Thu, 18 Jul 2013 07:41:11 -0700 Subject: [PATCH 2/6] Add mdb_reader_list() Dump the active slots in the reader table. --- libraries/liblmdb/lmdb.h | 17 +++++++++++++++++ libraries/liblmdb/mdb.c | 35 +++++++++++++++++++++++++++++++++++ libraries/liblmdb/mdb_stat.1 | 9 +++++++++ libraries/liblmdb/mdb_stat.c | 14 +++++++++++--- 4 files changed, 72 insertions(+), 3 deletions(-) diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h index c38c7dbf8f..af046d19c8 100644 --- a/libraries/liblmdb/lmdb.h +++ b/libraries/liblmdb/lmdb.h @@ -1298,6 +1298,23 @@ int mdb_cmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b); * @return < 0 if a < b, 0 if a == b, > 0 if a > b */ int mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b); + + /** @brief A callback function used to print a message from the library. + * + * @param[in] msg The string to be printed. + * @param[in] ctx An arbitrary context pointer for the callback. + * @return < 0 on failure, 0 on success. + */ +typedef int (MDB_msg_func)(const char *msg, void *ctx); + + /** @brief Dump the entries in the reader lock table. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] func A #MDB_msg_func function + * @param[in] ctx Anything the message function needs + * @return < 0 on failure, 0 on success. + */ +int mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx); /** @} */ #ifdef __cplusplus diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index f895c0fb5e..f86ec356b6 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -7916,4 +7916,39 @@ int mdb_set_relctx(MDB_txn *txn, MDB_dbi dbi, void *ctx) return MDB_SUCCESS; } +int mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx) +{ + unsigned int i, rdrs; + MDB_reader *mr; + char buf[128]; + int first = 1; + + if (!env || !func) + return -1; + if (!env->me_txns) { + return func("No reader locks\n", ctx); + } + rdrs = env->me_numreaders; + mr = env->me_txns->mti_readers; + for (i=0; i Date: Thu, 18 Jul 2013 08:33:24 -0700 Subject: [PATCH 3/6] Tweak mdb_stat(1) Don't obtain reader txn before displaying reader table. Exit after reader table if no other DB query options were given. --- libraries/liblmdb/mdb.c | 5 ++++- libraries/liblmdb/mdb_stat.c | 16 ++++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index f86ec356b6..e15b5bdb8e 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -7926,7 +7926,7 @@ int mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx) if (!env || !func) return -1; if (!env->me_txns) { - return func("No reader locks\n", ctx); + return func("(no reader locks)\n", ctx); } rdrs = env->me_numreaders; mr = env->me_txns->mti_readers; @@ -7949,6 +7949,9 @@ int mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx) return rc; } } + if (first) { + func("(no active readers)\n", ctx); + } return 0; } /** @} */ diff --git a/libraries/liblmdb/mdb_stat.c b/libraries/liblmdb/mdb_stat.c index 5a0946683d..8331bb9cde 100644 --- a/libraries/liblmdb/mdb_stat.c +++ b/libraries/liblmdb/mdb_stat.c @@ -31,7 +31,7 @@ static void prstat(MDB_stat *ms) static void usage(char *prog) { - fprintf(stderr, "usage: %s dbpath [-e] [-f[f[f]]] [-n] [-a|-s subdb] [-r]\n", prog); + fprintf(stderr, "usage: %s dbpath [-n] [-e] [-r] | [-f[f[f]]] [-a|-s subdb]\n", prog); exit(EXIT_FAILURE); } @@ -56,6 +56,7 @@ int main(int argc, char *argv[]) * -s: print stat of only the named subDB * -e: print env info * -f: print freelist info + * -r: print reader info * -n: use NOSUBDIR flag on env_open * (default) print stat of only the main DB */ @@ -103,11 +104,6 @@ int main(int argc, char *argv[]) printf("mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc)); goto env_close; } - rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); - if (rc) { - printf("mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc)); - goto env_close; - } if (envinfo) { rc = mdb_env_stat(env, &mst); @@ -126,6 +122,14 @@ int main(int argc, char *argv[]) if (rdrinfo) { printf("Reader Table Status\n"); rc = mdb_reader_list(env, (MDB_msg_func *)fputs, stdout); + if (!(subname || alldbs || freinfo)) + goto env_close; + } + + rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (rc) { + printf("mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; } if (freinfo) { From 68a0adc59fc8b5385b357f4e1f9bb484bbb95f83 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Thu, 18 Jul 2013 09:00:51 -0700 Subject: [PATCH 4/6] Tweak reader_list --- libraries/liblmdb/mdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index e15b5bdb8e..91007a187e 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -7920,7 +7920,7 @@ int mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx) { unsigned int i, rdrs; MDB_reader *mr; - char buf[128]; + char buf[64]; int first = 1; if (!env || !func) From 4d7c9e5bca70e08166cba3711df17a4f4e479cda Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Thu, 18 Jul 2013 09:11:09 -0700 Subject: [PATCH 5/6] Split MDB_VERSION to MDB_DATA/MDB_LOCK VERSION --- libraries/liblmdb/mdb.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 91007a187e..5d08b42d9c 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -344,8 +344,10 @@ static txnid_t mdb_debug_start; */ #define MDB_MAGIC 0xBEEFC0DE - /** The version number for a database's file format. */ -#define MDB_VERSION 1 + /** The version number for a database's datafile format. */ +#define MDB_DATA_VERSION 1 + /** The version number for a database's lockfile format. */ +#define MDB_LOCK_VERSION 1 /** @brief The maximum size of a key in the database. * @@ -513,7 +515,7 @@ typedef struct MDB_txbody { /** Stamp identifying this as an MDB file. It must be set * to #MDB_MAGIC. */ uint32_t mtb_magic; - /** Version number of this lock file. Must be set to #MDB_VERSION. */ + /** Version number of this lock file. Must be set to #MDB_LOCK_VERSION. */ uint32_t mtb_version; #if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) char mtb_rmname[MNAME_LEN]; @@ -770,7 +772,7 @@ typedef struct MDB_meta { /** Stamp identifying this as an MDB file. It must be set * to #MDB_MAGIC. */ uint32_t mm_magic; - /** Version number of this lock file. Must be set to #MDB_VERSION. */ + /** Version number of this lock file. Must be set to #MDB_DATA_VERSION. */ uint32_t mm_version; void *mm_address; /**< address for fixed mapping */ size_t mm_mapsize; /**< size of mmap region */ @@ -2862,9 +2864,9 @@ mdb_env_read_header(MDB_env *env, MDB_meta *meta) return MDB_INVALID; } - if (m->mm_version != MDB_VERSION) { + if (m->mm_version != MDB_DATA_VERSION) { DPRINTF("database is version %u, expected version %u", - m->mm_version, MDB_VERSION); + m->mm_version, MDB_DATA_VERSION); return MDB_VERSION_MISMATCH; } @@ -2891,7 +2893,7 @@ mdb_env_init_meta(MDB_env *env, MDB_meta *meta) GET_PAGESIZE(psize); meta->mm_magic = MDB_MAGIC; - meta->mm_version = MDB_VERSION; + meta->mm_version = MDB_DATA_VERSION; meta->mm_mapsize = env->me_mapsize; meta->mm_psize = psize; meta->mm_last_pg = 1; @@ -3652,7 +3654,7 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl) pthread_mutexattr_destroy(&mattr); #endif /* _WIN32 || MDB_USE_POSIX_SEM */ - env->me_txns->mti_version = MDB_VERSION; + env->me_txns->mti_version = MDB_LOCK_VERSION; env->me_txns->mti_magic = MDB_MAGIC; env->me_txns->mti_txnid = 0; env->me_txns->mti_numreaders = 0; @@ -3663,9 +3665,9 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl) rc = MDB_INVALID; goto fail; } - if (env->me_txns->mti_version != MDB_VERSION) { + if (env->me_txns->mti_version != MDB_LOCK_VERSION) { DPRINTF("lock region is version %u, expected version %u", - env->me_txns->mti_version, MDB_VERSION); + env->me_txns->mti_version, MDB_LOCK_VERSION); rc = MDB_VERSION_MISMATCH; goto fail; } From a4bbe57f8ba4c7d88ba6d16a63171c2ec2ce02ad Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Thu, 18 Jul 2013 10:40:21 -0700 Subject: [PATCH 6/6] Add mdb_reader_check() --- libraries/liblmdb/lmdb.h | 8 ++ libraries/liblmdb/mdb.c | 152 ++++++++++++++++++++++++++++++++++- libraries/liblmdb/mdb_stat.1 | 5 +- libraries/liblmdb/mdb_stat.c | 8 +- 4 files changed, 170 insertions(+), 3 deletions(-) diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h index af046d19c8..b3cd5ef79e 100644 --- a/libraries/liblmdb/lmdb.h +++ b/libraries/liblmdb/lmdb.h @@ -1315,6 +1315,14 @@ typedef int (MDB_msg_func)(const char *msg, void *ctx); * @return < 0 on failure, 0 on success. */ int mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx); + + /** @brief Check for stale entries in the reader lock table. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] dead Number of stale slots that were cleared + * @return 0 on success, non-zero on failure. + */ +int mdb_reader_check(MDB_env *env, int *dead); /** @} */ #ifdef __cplusplus diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 5d08b42d9c..b36ff2bb4f 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -952,6 +952,8 @@ struct MDB_env { #define MDB_ENV_ACTIVE 0x20000000U /** me_txkey is set */ #define MDB_ENV_TXKEY 0x10000000U + /** Have liveness lock in reader table */ +#define MDB_LIVE_READER 0x08000000U uint32_t me_flags; /**< @ref mdb_env */ unsigned int me_psize; /**< size of a page, from #GET_PAGESIZE */ unsigned int me_maxreaders; /**< size of the reader table */ @@ -983,6 +985,7 @@ struct MDB_env { /** Max size of a node on a page */ unsigned int me_nodemax; #ifdef _WIN32 + int me_pidquery; /**< Used in OpenProcess */ HANDLE me_rmutex; /* Windows mutexes don't reside in shared mem */ HANDLE me_wmutex; #elif defined(MDB_USE_POSIX_SEM) @@ -1994,6 +1997,56 @@ mdb_cursors_close(MDB_txn *txn, unsigned merge) static void mdb_txn_reset0(MDB_txn *txn, const char *act); +#ifdef _WIN32 +enum Pidlock_op { + Pidset, Pidcheck +}; +#else +enum Pidlock_op { + Pidset = F_SETLK, Pidcheck = F_GETLK +}; +#endif + +/** Set or check a pid lock. Set returns 0 on success. + * Check returns 0 if lock exists (meaning the process is alive). + * + * On Windows Pidset is a no-op, we merely check for the existence + * of the process with the given pid. On POSIX we use a single byte + * lock on the lockfile, set at an offset equal to the pid. + */ +static int +mdb_reader_pid(MDB_env *env, enum Pidlock_op op, pid_t pid) +{ +#ifdef _WIN32 + HANDLE h; + int ver, query; + switch(op) { + case Pidset: + break; + case Pidcheck: + h = OpenProcess(env->me_pidquery, FALSE, pid); + if (!h) + return GetLastError(); + CloseHandle(h); + break; + } + return 0; +#else + int rc; + struct flock lock_info; + memset((void *)&lock_info, 0, sizeof(lock_info)); + lock_info.l_type = F_WRLCK; + lock_info.l_whence = SEEK_SET; + lock_info.l_start = pid; + lock_info.l_len = 1; + while ((rc = fcntl(env->me_lfd, op, &lock_info)) && + (rc = ErrCode()) == EINTR) ; + if (op == F_GETLK && rc == 0 && lock_info.l_type == F_UNLCK) + rc = -1; + return rc; +#endif +} + /** Common code for #mdb_txn_begin() and #mdb_txn_renew(). * @param[in] txn the transaction handle to initialize * @return 0 on success, non-zero on failure. @@ -2033,6 +2086,14 @@ mdb_txn_renew0(MDB_txn *txn) UNLOCK_MUTEX_R(env); return MDB_READERS_FULL; } + if (!(env->me_flags & MDB_LIVE_READER)) { + rc = mdb_reader_pid(env, Pidset, pid); + if (rc) { + UNLOCK_MUTEX_R(env); + return rc; + } + env->me_flags |= MDB_LIVE_READER; + } env->me_txns->mti_readers[i].mr_pid = pid; env->me_txns->mti_readers[i].mr_tid = tid; if (i >= env->me_txns->mti_numreaders) @@ -3161,6 +3222,14 @@ mdb_env_open2(MDB_env *env) LONG sizelo, sizehi; sizelo = env->me_mapsize & 0xffffffff; sizehi = env->me_mapsize >> 16 >> 16; /* only needed on Win64 */ + + /* See if we should use QueryLimited */ + rc = GetVersion(); + if ((rc & 0xff) > 5) + env->me_pidquery = PROCESS_QUERY_LIMITED_INFORMATION; + else + env->me_pidquery = PROCESS_QUERY_INFORMATION; + /* Windows won't create mappings for zero length files. * Just allocate the maxsize right now. */ @@ -7930,7 +7999,7 @@ int mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx) if (!env->me_txns) { return func("(no reader locks)\n", ctx); } - rdrs = env->me_numreaders; + rdrs = env->me_maxreaders; mr = env->me_txns->mti_readers; for (i=0; i> 1; + cursor = base + pivot + 1; + val = pid - ids[cursor]; + + if( val < 0 ) { + n = pivot; + + } else if ( val > 0 ) { + base = cursor; + n -= pivot + 1; + + } else { + /* found, so it's a duplicate */ + return -1; + } + } + + if( val > 0 ) { + ++cursor; + } + ids[0]++; + for (n = ids[0]; n > cursor; n--) + ids[n] = ids[n-1]; + ids[n] = pid; + return 0; +} + +int mdb_reader_check(MDB_env *env, int *dead) +{ + unsigned int i, j, rdrs; + MDB_reader *mr; + pid_t *pids, pid; + int count = 0; + + if (!env) + return EINVAL; + if (dead) + *dead = 0; + if (!env->me_txns) + return MDB_SUCCESS; + rdrs = env->me_maxreaders; + pids = malloc((rdrs+1) * sizeof(pid_t)); + if (!pids) + return ENOMEM; + pids[0] = 0; + mr = env->me_txns->mti_readers; + j = 0; + for (i=0; ime_pid) { + pid = mr[i].mr_pid; + if (mdb_pid_insert(pids, pid) == 0) { + if (mdb_reader_pid(env, Pidcheck, pid)) { + LOCK_MUTEX_R(env); + for (j=i; j 1) { + int dead; + mdb_reader_check(env, &dead); + printf(" %d stale readers cleared.\n", dead); + rc = mdb_reader_list(env, (MDB_msg_func *)fputs, stdout); + } if (!(subname || alldbs || freinfo)) goto env_close; }