From 7ef970436d83a734f56ddf31f8868eb6032d666a Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Wed, 23 Jul 2014 17:10:41 -0700 Subject: [PATCH 1/6] Revert ITS#7789 commits, let's try this again. Revert "ITS#7789 Fix resize vs MDB_NOMETASYNC, and a comment." Revert "ITS#7789 more" Revert "Revert "ITS#7789 update RESIZED errmsg text"" Revert "ITS#7789 update RESIZED errmsg text" Revert "ITS#7789 persist mapsize changes" This reverts commit 02285aca58f5629547263ba09e7dd685dcf6b4b5. This reverts commit 38e71c68de2ad61fb0fd72fd40214fd9a94479b9. This reverts commit a5e4eecb1e59e9645113bb4c63d06a23a2c4bdf9. This reverts commit d8e18551c3f03d5205f4dc8d1711e494d3814b17. This reverts commit 5926e54bba7c976e0837221efb20cbff384207b8. --- libraries/liblmdb/lmdb.h | 7 ++-- libraries/liblmdb/mdb.c | 74 +++++++++++++--------------------------- 2 files changed, 26 insertions(+), 55 deletions(-) diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h index 2374ef7cc7..dfdbddc440 100644 --- a/libraries/liblmdb/lmdb.h +++ b/libraries/liblmdb/lmdb.h @@ -411,7 +411,7 @@ typedef enum MDB_cursor_op { #define MDB_CURSOR_FULL (-30787) /** Page has not enough space - internal error */ #define MDB_PAGE_FULL (-30786) - /** Environment mapsize was changed by another process */ + /** Database contents grew beyond environment mapsize */ #define MDB_MAP_RESIZED (-30785) /** MDB_INCOMPATIBLE: Operation and DB incompatible, or DB flags changed */ #define MDB_INCOMPATIBLE (-30784) @@ -785,10 +785,7 @@ int mdb_env_get_fd(MDB_env *env, mdb_filehandle_t *fd); * This function should be called after #mdb_env_create() and before #mdb_env_open(). * It may be called at later times if no transactions are active in * this process. Note that the library does not check for this condition, - * the caller must ensure it explicitly. The new size takes effect - * immediately for the current process but will not be persisted to - * any others until a write transaction has been committed by the - * current process. + * the caller must ensure it explicitly. * * If the mapsize is changed by another process, #mdb_txn_begin() will * return #MDB_MAP_RESIZED. This function may be called with a size diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 9e4a068941..cc4dd6ddc9 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -1074,8 +1074,6 @@ struct MDB_env { HANDLE me_mfd; /**< just for writing the meta pages */ /** Failed to update the meta page. Probably an I/O error. */ #define MDB_FATAL_ERROR 0x80000000U - /** We're explicitly changing the mapsize. */ -#define MDB_RESIZING 0x40000000U /** Some fields are initialized. */ #define MDB_ENV_ACTIVE 0x20000000U /** me_txkey is set */ @@ -2537,9 +2535,7 @@ mdb_txn_renew0(MDB_txn *txn) } txn->mt_dbflags[0] = txn->mt_dbflags[1] = DB_VALID; - /* If we didn't ask for a resize, but the size grew, fail */ - if (!(env->me_flags & MDB_RESIZING) - && env->me_mapsize < meta->mm_mapsize) { + if (env->me_maxpg < txn->mt_next_pgno) { mdb_txn_reset0(txn, "renew0-mapfail"); if (new_notls) { txn->mt_u.reader->mr_pid = 0; @@ -3277,13 +3273,8 @@ mdb_txn_commit(MDB_txn *txn) mdb_cursors_close(txn, 0); if (!txn->mt_u.dirty_list[0].mid && - !(txn->mt_flags & (MDB_TXN_DIRTY|MDB_TXN_SPILLS))) { - if ((env->me_flags & MDB_RESIZING) - && (rc = mdb_env_write_meta(txn))) { - goto fail; - } + !(txn->mt_flags & (MDB_TXN_DIRTY|MDB_TXN_SPILLS))) goto done; - } DPRINTF(("committing txn %"Z"u %p on mdbenv %p, root page %"Z"u", txn->mt_txnid, (void*)txn, (void*)env, txn->mt_dbs[MAIN_DBI].md_root)); @@ -3324,6 +3315,7 @@ mdb_txn_commit(MDB_txn *txn) #endif if ((rc = mdb_page_flush(txn, 0)) || + (rc = mdb_env_sync(env, 0)) || (rc = mdb_env_write_meta(txn))) goto fail; @@ -3480,7 +3472,7 @@ mdb_env_init_meta(MDB_env *env, MDB_meta *meta) static int mdb_env_write_meta(MDB_txn *txn) { - MDB_env *env = txn->mt_env; + MDB_env *env; MDB_meta meta, metab, *mp; off_t off; int rc, len, toggle; @@ -3492,22 +3484,17 @@ mdb_env_write_meta(MDB_txn *txn) int r2; #endif - /* Sync data and previous metapage before writing a new metapage */ - if ((rc = mdb_env_sync(env, 0)) != MDB_SUCCESS) - return rc; - toggle = txn->mt_txnid & 1; DPRINTF(("writing meta page %d for root page %"Z"u", toggle, txn->mt_dbs[MAIN_DBI].md_root)); + env = txn->mt_env; mp = env->me_metas[toggle]; if (env->me_flags & MDB_WRITEMAP) { - /* Persist any changes of mapsize config */ - if (env->me_flags & MDB_RESIZING) { + /* Persist any increases of mapsize config */ + if (env->me_mapsize > mp->mm_mapsize) mp->mm_mapsize = env->me_mapsize; - env->me_flags ^= MDB_RESIZING; - } mp->mm_dbs[0] = txn->mt_dbs[0]; mp->mm_dbs[1] = txn->mt_dbs[1]; mp->mm_last_pg = txn->mt_next_pgno - 1; @@ -3535,11 +3522,10 @@ mdb_env_write_meta(MDB_txn *txn) metab.mm_last_pg = env->me_metas[toggle]->mm_last_pg; ptr = (char *)&meta; - if (env->me_flags & MDB_RESIZING) { - /* Persist any changes of mapsize config */ + if (env->me_mapsize > mp->mm_mapsize) { + /* Persist any increases of mapsize config */ meta.mm_mapsize = env->me_mapsize; off = offsetof(MDB_meta, mm_mapsize); - env->me_flags ^= MDB_RESIZING; } else { off = offsetof(MDB_meta, mm_dbs[0].md_depth); } @@ -3728,25 +3714,19 @@ mdb_env_set_mapsize(MDB_env *env, size_t size) * sure there are no active txns. */ if (env->me_map) { - int rc, change = 0; + int rc; void *old; if (env->me_txn) return EINVAL; if (!size) size = env->me_metas[mdb_env_pick_meta(env)]->mm_mapsize; - else { - if (size < env->me_mapsize) { - /* If the configured size is smaller, make sure it's - * still big enough. Silently round up to minimum if not. - */ - size_t minsize = (env->me_metas[mdb_env_pick_meta(env)]->mm_last_pg + 1) * env->me_psize; - if (size < minsize) - size = minsize; - } - /* nothing actually changed */ - if (size == env->me_mapsize) - return MDB_SUCCESS; - change = 1; + else if (size < env->me_mapsize) { + /* If the configured size is smaller, make sure it's + * still big enough. Silently round up to minimum if not. + */ + size_t minsize = (env->me_metas[mdb_env_pick_meta(env)]->mm_last_pg + 1) * env->me_psize; + if (size < minsize) + size = minsize; } munmap(env->me_map, env->me_mapsize); env->me_mapsize = size; @@ -3754,8 +3734,6 @@ mdb_env_set_mapsize(MDB_env *env, size_t size) rc = mdb_env_map(env, old); if (rc) return rc; - if (change) - env->me_flags |= MDB_RESIZING; } env->me_mapsize = size; if (env->me_psize) @@ -3828,17 +3806,13 @@ mdb_env_open2(MDB_env *env) * else use the size recorded in the existing env. */ env->me_mapsize = newenv ? DEFAULT_MAPSIZE : meta.mm_mapsize; - } else { - if (env->me_mapsize < meta.mm_mapsize) { - /* If the configured size is smaller, make sure it's - * still big enough. Silently round up to minimum if not. - */ - size_t minsize = (meta.mm_last_pg + 1) * meta.mm_psize; - if (env->me_mapsize < minsize) - env->me_mapsize = minsize; - } - if (env->me_mapsize != meta.mm_mapsize) - env->me_flags |= MDB_RESIZING; + } else if (env->me_mapsize < meta.mm_mapsize) { + /* If the configured size is smaller, make sure it's + * still big enough. Silently round up to minimum if not. + */ + size_t minsize = (meta.mm_last_pg + 1) * meta.mm_psize; + if (env->me_mapsize < minsize) + env->me_mapsize = minsize; } rc = mdb_env_map(env, (flags & MDB_FIXEDMAP) ? meta.mm_address : NULL); From 81b502e17e4ed82d73c3c4b88d86b8533dbc4be7 Mon Sep 17 00:00:00 2001 From: Hallvard Furuseth Date: Wed, 23 Jul 2014 17:24:31 -0700 Subject: [PATCH 2/6] ITS#7789 Fix mdb_env_write_meta(). Always write mapsize = max(other metapage:mapsize, env:mapsize). --- libraries/liblmdb/mdb.c | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index cc4dd6ddc9..88120b90b4 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -3474,6 +3474,7 @@ mdb_env_write_meta(MDB_txn *txn) { MDB_env *env; MDB_meta meta, metab, *mp; + size_t mapsize; off_t off; int rc, len, toggle; char *ptr; @@ -3490,11 +3491,13 @@ mdb_env_write_meta(MDB_txn *txn) env = txn->mt_env; mp = env->me_metas[toggle]; + mapsize = env->me_metas[toggle ^ 1]->mm_mapsize; + /* Persist any increases of mapsize config */ + if (mapsize < env->me_mapsize) + mapsize = env->me_mapsize; if (env->me_flags & MDB_WRITEMAP) { - /* Persist any increases of mapsize config */ - if (env->me_mapsize > mp->mm_mapsize) - mp->mm_mapsize = env->me_mapsize; + mp->mm_mapsize = mapsize; mp->mm_dbs[0] = txn->mt_dbs[0]; mp->mm_dbs[1] = txn->mt_dbs[1]; mp->mm_last_pg = txn->mt_next_pgno - 1; @@ -3521,22 +3524,15 @@ mdb_env_write_meta(MDB_txn *txn) metab.mm_txnid = env->me_metas[toggle]->mm_txnid; metab.mm_last_pg = env->me_metas[toggle]->mm_last_pg; - ptr = (char *)&meta; - if (env->me_mapsize > mp->mm_mapsize) { - /* Persist any increases of mapsize config */ - meta.mm_mapsize = env->me_mapsize; - off = offsetof(MDB_meta, mm_mapsize); - } else { - off = offsetof(MDB_meta, mm_dbs[0].md_depth); - } - len = sizeof(MDB_meta) - off; - - ptr += off; + meta.mm_mapsize = mapsize; meta.mm_dbs[0] = txn->mt_dbs[0]; meta.mm_dbs[1] = txn->mt_dbs[1]; meta.mm_last_pg = txn->mt_next_pgno - 1; meta.mm_txnid = txn->mt_txnid; + off = offsetof(MDB_meta, mm_mapsize); + ptr = (char *)&meta + off; + len = sizeof(MDB_meta) - off; if (toggle) off += env->me_psize; off += PAGEHDRSZ; From 1b08d8e503d3ff7824af990d175a3b966b6b9e33 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Wed, 23 Jul 2014 17:31:34 -0700 Subject: [PATCH 3/6] ITS#7789 doc updates --- libraries/liblmdb/lmdb.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h index dfdbddc440..b48100a25a 100644 --- a/libraries/liblmdb/lmdb.h +++ b/libraries/liblmdb/lmdb.h @@ -787,7 +787,13 @@ int mdb_env_get_fd(MDB_env *env, mdb_filehandle_t *fd); * this process. Note that the library does not check for this condition, * the caller must ensure it explicitly. * - * If the mapsize is changed by another process, #mdb_txn_begin() will + * The new size takes effect immediately for the current process but + * will not be persisted to any others until a write transaction has been + * committed by the current process. Also, only mapsize increases are + * persisted into the environment. + * + * If the mapsize is increased by another process, and data has grown + * beyond the range of the current mapsize, #mdb_txn_begin() will * return #MDB_MAP_RESIZED. This function may be called with a size * of zero to adopt the new size. * From 6cb1166635de7812e57aa0a36c68a5c3e85b8284 Mon Sep 17 00:00:00 2001 From: Hallvard Furuseth Date: Wed, 9 Jul 2014 21:38:45 +0200 Subject: [PATCH 4/6] Fix mdb_copy & co when MDB pagesize < OS pagesize --- libraries/liblmdb/mdb.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 88120b90b4..3e62b2e28f 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -8410,13 +8410,13 @@ mdb_env_copyfd1(MDB_env *env, HANDLE fd) #ifdef _WIN32 my.mc_mutex = CreateMutex(NULL, FALSE, NULL); my.mc_cond = CreateEvent(NULL, FALSE, FALSE, NULL); - my.mc_wbuf[0] = _aligned_malloc(MDB_WBUF*2, env->me_psize); + my.mc_wbuf[0] = _aligned_malloc(MDB_WBUF*2, env->me_os_psize); if (my.mc_wbuf[0] == NULL) return errno; #else pthread_mutex_init(&my.mc_mutex, NULL); pthread_cond_init(&my.mc_cond, NULL); - rc = posix_memalign((void **)&my.mc_wbuf[0], env->me_psize, MDB_WBUF*2); + rc = posix_memalign((void **)&my.mc_wbuf[0], env->me_os_psize, MDB_WBUF*2); if (rc) return rc; #endif @@ -8654,6 +8654,7 @@ mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags) goto leave; } + if (env->me_psize >= env->me_os_psize) { #ifdef O_DIRECT /* Set O_DIRECT if the file system supports it */ if ((rc = fcntl(newfd, F_GETFL)) != -1) @@ -8666,6 +8667,7 @@ mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags) goto leave; } #endif + } rc = mdb_env_copyfd2(env, newfd, flags); From 082bac1492c1aec8ac0fbe8cc10ac3f46424b836 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Thu, 24 Jul 2014 04:13:55 -0700 Subject: [PATCH 5/6] more for mdb_cursor_count return MDB_NOTFOUND if the cursor is empty --- libraries/liblmdb/mdb.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 3e62b2e28f..84477c4285 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -6962,6 +6962,9 @@ mdb_cursor_count(MDB_cursor *mc, size_t *countp) if (!(mc->mc_flags & C_INITIALIZED)) return EINVAL; + if (!mc->mc_snum || (mc->mc_flags & C_EOF)) + return MDB_NOTFOUND; + leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { *countp = 1; From d098184f613791c0712a4c23584ddcb858054e42 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Thu, 24 Jul 2014 04:15:45 -0700 Subject: [PATCH 6/6] Update date --- libraries/liblmdb/lmdb.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h index b48100a25a..a4c4bca621 100644 --- a/libraries/liblmdb/lmdb.h +++ b/libraries/liblmdb/lmdb.h @@ -194,7 +194,7 @@ typedef int mdb_filehandle_t; MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH) /** The release date of this library version */ -#define MDB_VERSION_DATE "July 8, 2014" +#define MDB_VERSION_DATE "July 24, 2014" /** A stringifier for the version info */ #define MDB_VERSTR(a,b,c,d) "LMDB " #a "." #b "." #c ": (" d ")"