From 2f04728471384e8c1ea06d850590002d45deb3ad Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Tue, 8 Jul 2014 16:27:03 -0700 Subject: [PATCH 01/19] ITS#7892 Fix MacOSX section name --- libraries/liblmdb/mdb.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index d8cf710bfd..267a6e3951 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -152,7 +152,11 @@ #ifdef __GNUC__ /** Put infrequently used env functions in separate section */ -#define ESECT __attribute__ ((section("text_env"))) +# ifdef __APPLE__ +# define ESECT __attribute__ ((section("__TEXT,text_env"))) +# else +# define ESECT __attribute__ ((section("text_env"))) +# endif #else #define ESECT #endif From 604256ceffa2c3401ea6c15e756b3f0155f2c742 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Tue, 8 Jul 2014 16:27:26 -0700 Subject: [PATCH 02/19] Update LMDB version date/info --- libraries/liblmdb/CHANGES | 12 ++++++++++++ libraries/liblmdb/lmdb.h | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/libraries/liblmdb/CHANGES b/libraries/liblmdb/CHANGES index 49e4cabbfd..f16fb9527b 100644 --- a/libraries/liblmdb/CHANGES +++ b/libraries/liblmdb/CHANGES @@ -1,5 +1,17 @@ LMDB 0.9 Change Log +LMDB 0.9.14 Engineering + Fix to support 64K page size (ITS#7713) + Fix cursor bug when deleting last node of a DUPSORT key + Fix mdb_env_info to return FIXEDMAP address + Fix mdb_copy copying past end of file (ITS#7886) + Fix cursor bugs from page_merge/rebalance + Fix to dirty fewer pages in deletes (mdb_page_loose() + Fix Windows compat issues in mtests (ITS#7879) + Add compacting variant of mdb_copy + Add BigEndian integer key compare code + Add mdb_dump/mdb_load utilities + LMDB 0.9.13 Release (2014/06/18) Fix mdb_page_alloc unlimited overflow page search Documentation diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h index 0bc97cdf40..1a04682b1a 100644 --- a/libraries/liblmdb/lmdb.h +++ b/libraries/liblmdb/lmdb.h @@ -194,7 +194,7 @@ typedef int mdb_filehandle_t; MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH) /** The release date of this library version */ -#define MDB_VERSION_DATE "June 20, 2014" +#define MDB_VERSION_DATE "July 8, 2014" /** A stringifier for the version info */ #define MDB_VERSTR(a,b,c,d) "LMDB " #a "." #b "." #c ": (" d ")" From 5926e54bba7c976e0837221efb20cbff384207b8 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Tue, 8 Jul 2014 19:06:45 -0700 Subject: [PATCH 03/19] ITS#7789 persist mapsize changes Write decreases too, not just increases. Check for any size change that was not requested by this process. --- libraries/liblmdb/lmdb.h | 5 ++++- libraries/liblmdb/mdb.c | 48 ++++++++++++++++++++++++++++------------ 2 files changed, 38 insertions(+), 15 deletions(-) diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h index 1a04682b1a..068088ccd0 100644 --- a/libraries/liblmdb/lmdb.h +++ b/libraries/liblmdb/lmdb.h @@ -782,7 +782,10 @@ int mdb_env_get_fd(MDB_env *env, mdb_filehandle_t *fd); * This function should be called after #mdb_env_create() and before #mdb_env_open(). * It may be called at later times if no transactions are active in * this process. Note that the library does not check for this condition, - * the caller must ensure it explicitly. + * the caller must ensure it explicitly. The new size takes effect + * immediately for the current process but will not be persisted to + * any others until a write transaction has been committed by the + * current process. * * If the mapsize is changed by another process, #mdb_txn_begin() will * return #MDB_MAP_RESIZED. This function may be called with a size diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 267a6e3951..fd48ff136f 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -1071,6 +1071,8 @@ struct MDB_env { HANDLE me_mfd; /**< just for writing the meta pages */ /** Failed to update the meta page. Probably an I/O error. */ #define MDB_FATAL_ERROR 0x80000000U + /** We're explicitly changing the mapsize. */ +#define MDB_RESIZING 0x40000000U /** Some fields are initialized. */ #define MDB_ENV_ACTIVE 0x20000000U /** me_txkey is set */ @@ -2492,7 +2494,9 @@ mdb_txn_renew0(MDB_txn *txn) } txn->mt_dbflags[0] = txn->mt_dbflags[1] = DB_VALID; - if (env->me_maxpg < txn->mt_next_pgno) { + /* If we didn't ask for a resize, but the size changed, fail */ + if (!(env->me_flags & MDB_RESIZING) + && env->me_mapsize != meta->mm_mapsize) { mdb_txn_reset0(txn, "renew0-mapfail"); if (new_notls) { txn->mt_u.reader->mr_pid = 0; @@ -3216,8 +3220,13 @@ mdb_txn_commit(MDB_txn *txn) mdb_cursors_close(txn, 0); if (!txn->mt_u.dirty_list[0].mid && - !(txn->mt_flags & (MDB_TXN_DIRTY|MDB_TXN_SPILLS))) + !(txn->mt_flags & (MDB_TXN_DIRTY|MDB_TXN_SPILLS))) { + if ((env->me_flags & MDB_RESIZING) + && (rc = mdb_env_write_meta(txn))) { + goto fail; + } goto done; + } DPRINTF(("committing txn %"Z"u %p on mdbenv %p, root page %"Z"u", txn->mt_txnid, (void*)txn, (void*)env, txn->mt_dbs[MAIN_DBI].md_root)); @@ -3431,9 +3440,11 @@ mdb_env_write_meta(MDB_txn *txn) mp = env->me_metas[toggle]; if (env->me_flags & MDB_WRITEMAP) { - /* Persist any increases of mapsize config */ - if (env->me_mapsize > mp->mm_mapsize) + /* Persist any changes of mapsize config */ + if (env->me_flags & MDB_RESIZING) { mp->mm_mapsize = env->me_mapsize; + env->me_flags ^= MDB_RESIZING; + } mp->mm_dbs[0] = txn->mt_dbs[0]; mp->mm_dbs[1] = txn->mt_dbs[1]; mp->mm_last_pg = txn->mt_next_pgno - 1; @@ -3461,10 +3472,11 @@ mdb_env_write_meta(MDB_txn *txn) metab.mm_last_pg = env->me_metas[toggle]->mm_last_pg; ptr = (char *)&meta; - if (env->me_mapsize > mp->mm_mapsize) { - /* Persist any increases of mapsize config */ + if (env->me_flags & MDB_RESIZING) { + /* Persist any changes of mapsize config */ meta.mm_mapsize = env->me_mapsize; off = offsetof(MDB_meta, mm_mapsize); + env->me_flags ^= MDB_RESIZING; } else { off = offsetof(MDB_meta, mm_dbs[0].md_depth); } @@ -3652,19 +3664,25 @@ mdb_env_set_mapsize(MDB_env *env, size_t size) * sure there are no active txns. */ if (env->me_map) { - int rc; + int rc, change = 0; void *old; if (env->me_txn) return EINVAL; if (!size) size = env->me_metas[mdb_env_pick_meta(env)]->mm_mapsize; - else if (size < env->me_mapsize) { - /* If the configured size is smaller, make sure it's - * still big enough. Silently round up to minimum if not. - */ - size_t minsize = (env->me_metas[mdb_env_pick_meta(env)]->mm_last_pg + 1) * env->me_psize; - if (size < minsize) - size = minsize; + else { + if (size < env->me_mapsize) { + /* If the configured size is smaller, make sure it's + * still big enough. Silently round up to minimum if not. + */ + size_t minsize = (env->me_metas[mdb_env_pick_meta(env)]->mm_last_pg + 1) * env->me_psize; + if (size < minsize) + size = minsize; + } + /* nothing actually changed */ + if (size == env->me_mapsize) + return MDB_SUCCESS; + change = 1; } munmap(env->me_map, env->me_mapsize); env->me_mapsize = size; @@ -3672,6 +3690,8 @@ mdb_env_set_mapsize(MDB_env *env, size_t size) rc = mdb_env_map(env, old, 1); if (rc) return rc; + if (change) + env->me_flags |= MDB_RESIZING; } env->me_mapsize = size; if (env->me_psize) From d8e18551c3f03d5205f4dc8d1711e494d3814b17 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Tue, 8 Jul 2014 20:05:31 -0700 Subject: [PATCH 04/19] ITS#7789 update RESIZED errmsg text --- libraries/liblmdb/mdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index fd48ff136f..1e139aae0d 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -1241,7 +1241,7 @@ static char *const mdb_errstr[] = { "MDB_TXN_FULL: Transaction has too many dirty pages - transaction too big", "MDB_CURSOR_FULL: Internal error - cursor stack limit reached", "MDB_PAGE_FULL: Internal error - page has no more space", - "MDB_MAP_RESIZED: Database contents grew beyond environment mapsize", + "MDB_MAP_RESIZED: Environment mapsize was changed by another process", "MDB_INCOMPATIBLE: Operation and DB incompatible, or DB flags changed", "MDB_BAD_RSLOT: Invalid reuse of reader locktable slot", "MDB_BAD_TXN: Transaction cannot recover - it must be aborted", From 0401f2deed75a83d2de790b8a1313e1792e5a04f Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Tue, 8 Jul 2014 21:02:40 -0700 Subject: [PATCH 05/19] ITS#7825 add MDB_BAD_DBI error code Use DBI sequence numbers to detect DBIs being closed while in use. --- libraries/liblmdb/lmdb.h | 5 ++++- libraries/liblmdb/mdb.c | 34 ++++++++++++++++++++++++++++++---- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h index 068088ccd0..f63c6719e0 100644 --- a/libraries/liblmdb/lmdb.h +++ b/libraries/liblmdb/lmdb.h @@ -421,7 +421,10 @@ typedef enum MDB_cursor_op { #define MDB_BAD_TXN (-30782) /** Unsupported size of key/DB name/data, or wrong DUPFIXED size */ #define MDB_BAD_VALSIZE (-30781) -#define MDB_LAST_ERRCODE MDB_BAD_VALSIZE + /** The specified DBI was changed unexpectedly */ +#define MDB_BAD_DBI (-30780) + /** The last defined error code */ +#define MDB_LAST_ERRCODE MDB_BAD_DBI /** @} */ /** @brief Statistics for a database in the environment */ diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 1e139aae0d..0a6f3ab3f5 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -954,6 +954,8 @@ struct MDB_txn { MDB_dbx *mt_dbxs; /** Array of MDB_db records for each known DB */ MDB_db *mt_dbs; + /** Array of sequence numbers for each DB handle */ + unsigned int *mt_dbiseqs; /** @defgroup mt_dbflag Transaction DB Flags * @ingroup internal * @{ @@ -1096,6 +1098,7 @@ struct MDB_env { pgno_t me_maxpg; /**< me_mapsize / me_psize */ MDB_dbx *me_dbxs; /**< array of static DB info */ uint16_t *me_dbflags; /**< array of flags from MDB_db.md_flags */ + unsigned int *me_dbiseqs; /**< array of dbi sequence numbers */ pthread_key_t me_txkey; /**< thread-key for readers */ MDB_pgstate me_pgstate; /**< state of old pages from freeDB */ # define me_pglast me_pgstate.mf_pglast @@ -1145,6 +1148,10 @@ typedef struct MDB_ntxn { #define TXN_DBI_EXIST(txn, dbi) \ ((txn) && (dbi) < (txn)->mt_numdbs && ((txn)->mt_dbflags[dbi] & DB_VALID)) + /** Check for misused \b dbi handles */ +#define TXN_DBI_CHANGED(txn, md_name, dbi) \ + (!(md_name).mv_size || (txn)->mt_dbiseqs[dbi] != (txn)->mt_env->me_dbiseqs[dbi]) + static int mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp); static int mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp); static int mdb_page_touch(MDB_cursor *mc); @@ -1246,6 +1253,7 @@ static char *const mdb_errstr[] = { "MDB_BAD_RSLOT: Invalid reuse of reader locktable slot", "MDB_BAD_TXN: Transaction cannot recover - it must be aborted", "MDB_BAD_VALSIZE: Unsupported size of key/DB name/data, or wrong DUPFIXED size", + "MDB_BAD_DBI: The specified DBI handle was closed/changed unexpectedly", }; char * @@ -2406,6 +2414,7 @@ mdb_txn_renew0(MDB_txn *txn) /* Setup db info */ txn->mt_numdbs = env->me_numdbs; txn->mt_dbxs = env->me_dbxs; /* mostly static anyway */ + memcpy(txn->mt_dbiseqs, env->me_dbiseqs, env->me_maxdbs * sizeof(unsigned int)); if (txn->mt_flags & MDB_TXN_RDONLY) { if (!ti) { @@ -2554,7 +2563,7 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret) } tsize = sizeof(MDB_ntxn); } - size = tsize + env->me_maxdbs * (sizeof(MDB_db)+1); + size = tsize + env->me_maxdbs * (sizeof(MDB_db)+sizeof(unsigned int)+1); if (!(flags & MDB_RDONLY)) size += env->me_maxdbs * sizeof(MDB_cursor *); @@ -2563,11 +2572,12 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret) return ENOMEM; } txn->mt_dbs = (MDB_db *) ((char *)txn + tsize); + txn->mt_dbiseqs = (unsigned int *) (txn->mt_dbs + env->me_maxdbs); if (flags & MDB_RDONLY) { txn->mt_flags |= MDB_TXN_RDONLY; - txn->mt_dbflags = (unsigned char *)(txn->mt_dbs + env->me_maxdbs); + txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs); } else { - txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs); + txn->mt_cursors = (MDB_cursor **)(txn->mt_dbiseqs + env->me_maxdbs); txn->mt_dbflags = (unsigned char *)(txn->mt_cursors + env->me_maxdbs); } txn->mt_env = env; @@ -2651,6 +2661,7 @@ mdb_dbis_update(MDB_txn *txn, int keep) env->me_dbxs[i].md_name.mv_data = NULL; env->me_dbxs[i].md_name.mv_size = 0; env->me_dbflags[i] = 0; + env->me_dbiseqs[i]++; free(ptr); } } @@ -3241,6 +3252,10 @@ mdb_txn_commit(MDB_txn *txn) mdb_cursor_init(&mc, txn, MAIN_DBI, NULL); for (i = 2; i < txn->mt_numdbs; i++) { if (txn->mt_dbflags[i] & DB_DIRTY) { + if (TXN_DBI_CHANGED(txn, txn->mt_dbxs[i].md_name, i)) { + rc = MDB_BAD_DBI; + goto fail; + } data.mv_data = &txn->mt_dbs[i]; rc = mdb_cursor_put(&mc, &txn->mt_dbxs[i].md_name, &data, 0); if (rc) @@ -4344,7 +4359,8 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode env->me_path = strdup(path); env->me_dbxs = calloc(env->me_maxdbs, sizeof(MDB_dbx)); env->me_dbflags = calloc(env->me_maxdbs, sizeof(uint16_t)); - if (!(env->me_dbxs && env->me_path && env->me_dbflags)) { + env->me_dbiseqs = calloc(env->me_maxdbs, sizeof(unsigned int)); + if (!(env->me_dbxs && env->me_path && env->me_dbflags && env->me_dbiseqs)) { rc = ENOMEM; goto leave; } @@ -4440,6 +4456,7 @@ mdb_env_close0(MDB_env *env, int excl) free(env->me_dbxs[i].md_name.mv_data); free(env->me_pbuf); + free(env->me_dbiseqs); free(env->me_dbflags); free(env->me_dbxs); free(env->me_path); @@ -4959,6 +4976,8 @@ mdb_page_search(MDB_cursor *mc, MDB_val *key, int flags) /* Make sure we're using an up-to-date root */ if (*mc->mc_dbflag & DB_STALE) { MDB_cursor mc2; + if (TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbx->md_name, mc->mc_dbi)) + return MDB_BAD_DBI; mdb_cursor_init(&mc2, mc->mc_txn, MAIN_DBI, NULL); rc = mdb_page_search(&mc2, &mc->mc_dbx->md_name, 0); if (rc) @@ -5813,6 +5832,8 @@ mdb_cursor_touch(MDB_cursor *mc) if (mc->mc_dbi > MAIN_DBI && !(*mc->mc_dbflag & DB_DIRTY)) { MDB_cursor mc2; MDB_xcursor mcx; + if (TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbx->md_name, mc->mc_dbi)) + return MDB_BAD_DBI; mdb_cursor_init(&mc2, mc->mc_txn, MAIN_DBI, &mcx); rc = mdb_page_search(&mc2, &mc->mc_dbx->md_name, MDB_PS_MODIFY); if (rc) @@ -8872,6 +8893,7 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db txn->mt_dbxs[slot].md_name.mv_size = len; txn->mt_dbxs[slot].md_rel = NULL; txn->mt_dbflags[slot] = dbflag; + txn->mt_dbiseqs[slot] = ++txn->mt_env->me_dbiseqs[slot]; memcpy(&txn->mt_dbs[slot], data.mv_data, sizeof(MDB_db)); *dbi = slot; mdb_default_cmp(txn, slot); @@ -8909,6 +8931,7 @@ void mdb_dbi_close(MDB_env *env, MDB_dbi dbi) env->me_dbxs[dbi].md_name.mv_data = NULL; env->me_dbxs[dbi].md_name.mv_size = 0; env->me_dbflags[dbi] = 0; + env->me_dbiseqs[dbi]++; free(ptr); } @@ -9019,6 +9042,9 @@ int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del) if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) return EACCES; + if (dbi > MAIN_DBI && TXN_DBI_CHANGED(txn, txn->mt_dbxs[dbi].md_name, dbi)) + return MDB_BAD_DBI; + rc = mdb_cursor_open(txn, dbi, &mc); if (rc) return rc; From 04851e442f8939c38175b9742af5d62a19e1ee30 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Tue, 8 Jul 2014 21:22:07 -0700 Subject: [PATCH 06/19] ITS#7825 tweak prev commit dbiseqs are only checked in write txns so omit from read txns --- libraries/liblmdb/mdb.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 0a6f3ab3f5..c920149611 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -2414,7 +2414,6 @@ mdb_txn_renew0(MDB_txn *txn) /* Setup db info */ txn->mt_numdbs = env->me_numdbs; txn->mt_dbxs = env->me_dbxs; /* mostly static anyway */ - memcpy(txn->mt_dbiseqs, env->me_dbiseqs, env->me_maxdbs * sizeof(unsigned int)); if (txn->mt_flags & MDB_TXN_RDONLY) { if (!ti) { @@ -2488,6 +2487,7 @@ mdb_txn_renew0(MDB_txn *txn) txn->mt_free_pgs[0] = 0; txn->mt_spill_pgs = NULL; env->me_txn = txn; + memcpy(txn->mt_dbiseqs, env->me_dbiseqs, env->me_maxdbs * sizeof(unsigned int)); } /* Copy the DB info and flags */ @@ -2563,22 +2563,22 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret) } tsize = sizeof(MDB_ntxn); } - size = tsize + env->me_maxdbs * (sizeof(MDB_db)+sizeof(unsigned int)+1); + size = tsize + env->me_maxdbs * (sizeof(MDB_db)+1); if (!(flags & MDB_RDONLY)) - size += env->me_maxdbs * sizeof(MDB_cursor *); + size += env->me_maxdbs * (sizeof(MDB_cursor *)+sizeof(unsigned int)); if ((txn = calloc(1, size)) == NULL) { DPRINTF(("calloc: %s", strerror(ErrCode()))); return ENOMEM; } txn->mt_dbs = (MDB_db *) ((char *)txn + tsize); - txn->mt_dbiseqs = (unsigned int *) (txn->mt_dbs + env->me_maxdbs); if (flags & MDB_RDONLY) { txn->mt_flags |= MDB_TXN_RDONLY; - txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs); + txn->mt_dbflags = (unsigned char *)(txn->mt_dbs + env->me_maxdbs); } else { - txn->mt_cursors = (MDB_cursor **)(txn->mt_dbiseqs + env->me_maxdbs); - txn->mt_dbflags = (unsigned char *)(txn->mt_cursors + env->me_maxdbs); + txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs); + txn->mt_dbiseqs = (unsigned int *)(txn->mt_cursors + env->me_maxdbs); + txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs); } txn->mt_env = env; From 276aa559698ff3eca42bc855237296c29efad4fd Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Tue, 8 Jul 2014 21:31:48 -0700 Subject: [PATCH 07/19] ITS#7825 more only update dbiseq in dbi_open on write txns --- libraries/liblmdb/mdb.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index c920149611..0520eb72d5 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -8893,7 +8893,9 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db txn->mt_dbxs[slot].md_name.mv_size = len; txn->mt_dbxs[slot].md_rel = NULL; txn->mt_dbflags[slot] = dbflag; - txn->mt_dbiseqs[slot] = ++txn->mt_env->me_dbiseqs[slot]; + /* read txns don't track sequence numbers */ + if (!(txn->mt_flags & MDB_TXN_RDONLY)) + txn->mt_dbiseqs[slot] = ++txn->mt_env->me_dbiseqs[slot]; memcpy(&txn->mt_dbs[slot], data.mv_data, sizeof(MDB_db)); *dbi = slot; mdb_default_cmp(txn, slot); From e0273020bd1a14956fee04094d8380f8b30a13e5 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Tue, 8 Jul 2014 21:40:11 -0700 Subject: [PATCH 08/19] Updates #7789, #7825 --- libraries/liblmdb/CHANGES | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libraries/liblmdb/CHANGES b/libraries/liblmdb/CHANGES index f16fb9527b..6eac674b69 100644 --- a/libraries/liblmdb/CHANGES +++ b/libraries/liblmdb/CHANGES @@ -2,8 +2,10 @@ LMDB 0.9 Change Log LMDB 0.9.14 Engineering Fix to support 64K page size (ITS#7713) + Fix to persist decreased as well as increased mapsizes (ITS#7789) Fix cursor bug when deleting last node of a DUPSORT key Fix mdb_env_info to return FIXEDMAP address + Fix ambiguous error code from writing to closed DBI (ITS#7825) Fix mdb_copy copying past end of file (ITS#7886) Fix cursor bugs from page_merge/rebalance Fix to dirty fewer pages in deletes (mdb_page_loose() From d5371347e8a4ee5d5861958532007e7467c87a59 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Wed, 9 Jul 2014 05:13:10 -0700 Subject: [PATCH 09/19] ITS#7825 more Set read txn's dbiseqs to env's. Set child txn's dbiseqs to parent's. Simplify DBI_CHANGED test, no need to check md_name. No-op dbi_close of already closed handle. --- libraries/liblmdb/mdb.c | 66 ++++++++++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 24 deletions(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 0520eb72d5..96e37bcbb8 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -1149,8 +1149,8 @@ typedef struct MDB_ntxn { ((txn) && (dbi) < (txn)->mt_numdbs && ((txn)->mt_dbflags[dbi] & DB_VALID)) /** Check for misused \b dbi handles */ -#define TXN_DBI_CHANGED(txn, md_name, dbi) \ - (!(md_name).mv_size || (txn)->mt_dbiseqs[dbi] != (txn)->mt_env->me_dbiseqs[dbi]) +#define TXN_DBI_CHANGED(txn, dbi) \ + ((txn)->mt_dbiseqs[dbi] != (txn)->mt_env->me_dbiseqs[dbi]) static int mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp); static int mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp); @@ -2564,8 +2564,12 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret) tsize = sizeof(MDB_ntxn); } size = tsize + env->me_maxdbs * (sizeof(MDB_db)+1); - if (!(flags & MDB_RDONLY)) - size += env->me_maxdbs * (sizeof(MDB_cursor *)+sizeof(unsigned int)); + if (!(flags & MDB_RDONLY)) { + size += env->me_maxdbs * sizeof(MDB_cursor *); + /* child txns use parent's dbiseqs */ + if (!parent) + size += env->me_maxdbs * sizeof(unsigned int); + } if ((txn = calloc(1, size)) == NULL) { DPRINTF(("calloc: %s", strerror(ErrCode()))); @@ -2575,10 +2579,16 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret) if (flags & MDB_RDONLY) { txn->mt_flags |= MDB_TXN_RDONLY; txn->mt_dbflags = (unsigned char *)(txn->mt_dbs + env->me_maxdbs); + txn->mt_dbiseqs = env->me_dbiseqs; } else { txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs); - txn->mt_dbiseqs = (unsigned int *)(txn->mt_cursors + env->me_maxdbs); - txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs); + if (parent) { + txn->mt_dbiseqs = parent->mt_dbiseqs; + txn->mt_dbflags = (unsigned char *)(txn->mt_cursors + env->me_maxdbs); + } else { + txn->mt_dbiseqs = (unsigned int *)(txn->mt_cursors + env->me_maxdbs); + txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs); + } } txn->mt_env = env; @@ -2658,11 +2668,13 @@ mdb_dbis_update(MDB_txn *txn, int keep) env->me_dbflags[i] = txn->mt_dbs[i].md_flags | MDB_VALID; } else { char *ptr = env->me_dbxs[i].md_name.mv_data; - env->me_dbxs[i].md_name.mv_data = NULL; - env->me_dbxs[i].md_name.mv_size = 0; - env->me_dbflags[i] = 0; - env->me_dbiseqs[i]++; - free(ptr); + if (ptr) { + env->me_dbxs[i].md_name.mv_data = NULL; + env->me_dbxs[i].md_name.mv_size = 0; + env->me_dbflags[i] = 0; + env->me_dbiseqs[i]++; + free(ptr); + } } } } @@ -3252,7 +3264,7 @@ mdb_txn_commit(MDB_txn *txn) mdb_cursor_init(&mc, txn, MAIN_DBI, NULL); for (i = 2; i < txn->mt_numdbs; i++) { if (txn->mt_dbflags[i] & DB_DIRTY) { - if (TXN_DBI_CHANGED(txn, txn->mt_dbxs[i].md_name, i)) { + if (TXN_DBI_CHANGED(txn, i)) { rc = MDB_BAD_DBI; goto fail; } @@ -4976,7 +4988,7 @@ mdb_page_search(MDB_cursor *mc, MDB_val *key, int flags) /* Make sure we're using an up-to-date root */ if (*mc->mc_dbflag & DB_STALE) { MDB_cursor mc2; - if (TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbx->md_name, mc->mc_dbi)) + if (TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbi)) return MDB_BAD_DBI; mdb_cursor_init(&mc2, mc->mc_txn, MAIN_DBI, NULL); rc = mdb_page_search(&mc2, &mc->mc_dbx->md_name, 0); @@ -5832,7 +5844,7 @@ mdb_cursor_touch(MDB_cursor *mc) if (mc->mc_dbi > MAIN_DBI && !(*mc->mc_dbflag & DB_DIRTY)) { MDB_cursor mc2; MDB_xcursor mcx; - if (TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbx->md_name, mc->mc_dbi)) + if (TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbi)) return MDB_BAD_DBI; mdb_cursor_init(&mc2, mc->mc_txn, MAIN_DBI, &mcx); rc = mdb_page_search(&mc2, &mc->mc_dbx->md_name, MDB_PS_MODIFY); @@ -8808,7 +8820,7 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db MDB_dbi i; MDB_cursor mc; int rc, dbflag, exact; - unsigned int unused = 0; + unsigned int unused = 0, seq; size_t len; if (txn->mt_dbxs[FREE_DBI].md_cmp == NULL) { @@ -8893,9 +8905,12 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db txn->mt_dbxs[slot].md_name.mv_size = len; txn->mt_dbxs[slot].md_rel = NULL; txn->mt_dbflags[slot] = dbflag; - /* read txns don't track sequence numbers */ - if (!(txn->mt_flags & MDB_TXN_RDONLY)) - txn->mt_dbiseqs[slot] = ++txn->mt_env->me_dbiseqs[slot]; + /* txn-> and env-> are the same in read txns, use + * tmp variable to avoid undefined assignment + */ + seq = ++txn->mt_env->me_dbiseqs[slot]; + txn->mt_dbiseqs[slot] = seq; + memcpy(&txn->mt_dbs[slot], data.mv_data, sizeof(MDB_db)); *dbi = slot; mdb_default_cmp(txn, slot); @@ -8930,11 +8945,14 @@ void mdb_dbi_close(MDB_env *env, MDB_dbi dbi) if (dbi <= MAIN_DBI || dbi >= env->me_maxdbs) return; ptr = env->me_dbxs[dbi].md_name.mv_data; - env->me_dbxs[dbi].md_name.mv_data = NULL; - env->me_dbxs[dbi].md_name.mv_size = 0; - env->me_dbflags[dbi] = 0; - env->me_dbiseqs[dbi]++; - free(ptr); + /* If there was no name, this was already closed */ + if (ptr) { + env->me_dbxs[dbi].md_name.mv_data = NULL; + env->me_dbxs[dbi].md_name.mv_size = 0; + env->me_dbflags[dbi] = 0; + env->me_dbiseqs[dbi]++; + free(ptr); + } } int mdb_dbi_flags(MDB_txn *txn, MDB_dbi dbi, unsigned int *flags) @@ -9044,7 +9062,7 @@ int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del) if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) return EACCES; - if (dbi > MAIN_DBI && TXN_DBI_CHANGED(txn, txn->mt_dbxs[dbi].md_name, dbi)) + if (dbi > MAIN_DBI && TXN_DBI_CHANGED(txn, dbi)) return MDB_BAD_DBI; rc = mdb_cursor_open(txn, dbi, &mc); From a5e4eecb1e59e9645113bb4c63d06a23a2c4bdf9 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Wed, 9 Jul 2014 06:17:54 -0700 Subject: [PATCH 10/19] Revert "ITS#7789 update RESIZED errmsg text" This reverts commit d8e18551c3f03d5205f4dc8d1711e494d3814b17. And partially reverts mapsize check. Only mapsize increases are relevant. --- libraries/liblmdb/mdb.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 96e37bcbb8..d6ad1142b1 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -1248,7 +1248,7 @@ static char *const mdb_errstr[] = { "MDB_TXN_FULL: Transaction has too many dirty pages - transaction too big", "MDB_CURSOR_FULL: Internal error - cursor stack limit reached", "MDB_PAGE_FULL: Internal error - page has no more space", - "MDB_MAP_RESIZED: Environment mapsize was changed by another process", + "MDB_MAP_RESIZED: Database contents grew beyond environment mapsize", "MDB_INCOMPATIBLE: Operation and DB incompatible, or DB flags changed", "MDB_BAD_RSLOT: Invalid reuse of reader locktable slot", "MDB_BAD_TXN: Transaction cannot recover - it must be aborted", @@ -2503,9 +2503,9 @@ mdb_txn_renew0(MDB_txn *txn) } txn->mt_dbflags[0] = txn->mt_dbflags[1] = DB_VALID; - /* If we didn't ask for a resize, but the size changed, fail */ + /* If we didn't ask for a resize, but the size grew, fail */ if (!(env->me_flags & MDB_RESIZING) - && env->me_mapsize != meta->mm_mapsize) { + && env->me_mapsize < meta->mm_mapsize) { mdb_txn_reset0(txn, "renew0-mapfail"); if (new_notls) { txn->mt_u.reader->mr_pid = 0; From 38e71c68de2ad61fb0fd72fd40214fd9a94479b9 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Wed, 9 Jul 2014 06:54:32 -0700 Subject: [PATCH 11/19] ITS#7789 more persist size changes from env_open() as well. This was the original behavior. --- libraries/liblmdb/mdb.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index d6ad1142b1..c2f2c7439b 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -3791,13 +3791,17 @@ mdb_env_open2(MDB_env *env) * else use the size recorded in the existing env. */ env->me_mapsize = newenv ? DEFAULT_MAPSIZE : meta.mm_mapsize; - } else if (env->me_mapsize < meta.mm_mapsize) { - /* If the configured size is smaller, make sure it's - * still big enough. Silently round up to minimum if not. - */ - size_t minsize = (meta.mm_last_pg + 1) * meta.mm_psize; - if (env->me_mapsize < minsize) - env->me_mapsize = minsize; + } else { + if (env->me_mapsize < meta.mm_mapsize) { + /* If the configured size is smaller, make sure it's + * still big enough. Silently round up to minimum if not. + */ + size_t minsize = (meta.mm_last_pg + 1) * meta.mm_psize; + if (env->me_mapsize < minsize) + env->me_mapsize = minsize; + } + if (env->me_mapsize != meta.mm_mapsize) + env->me_flags |= MDB_RESIZING; } rc = mdb_env_map(env, meta.mm_address, newenv || env->me_mapsize != meta.mm_mapsize); From c4d0f90fce067aa02c91b975810bcd4391451430 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Wed, 9 Jul 2014 09:16:03 -0700 Subject: [PATCH 12/19] Fix loading of printable dump Off-by-one in end check, would complain on valid input. --- libraries/liblmdb/mdb_load.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c index 17f4757330..7a76d3c005 100644 --- a/libraries/liblmdb/mdb_load.c +++ b/libraries/liblmdb/mdb_load.c @@ -239,7 +239,7 @@ badend: if (c2[1] == '\\') { c1++; c2 += 2; } else { - if (c2+3 >= end || !isxdigit(c2[1]) || !isxdigit(c2[2])) { + if (c2+3 > end || !isxdigit(c2[1]) || !isxdigit(c2[2])) { Eof = 1; badend(); return EOF; From f3573a73831ee05d7d36dd4a5d89df5cbe2e4828 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Wed, 9 Jul 2014 09:31:49 -0700 Subject: [PATCH 13/19] Windows compat We mainly use Win32 functions, so cannot just use C-runtime strerror to return error messages. We have to use Win32-specific messages. Unfortunately, we document the API to return C-runtime error codes, so we can't just switch to all Win32 error codes. --- libraries/liblmdb/mdb.c | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index c2f2c7439b..4712ac25d6 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -1259,6 +1259,14 @@ static char *const mdb_errstr[] = { char * mdb_strerror(int err) { +#ifdef _WIN32 + /** HACK: pad 4KB on stack over the buf. Return system msgs in buf. + * This works as long as no function between the call to mdb_strerror + * and the actual use of the message uses more than 4K of stack. + */ + char pad[4096]; + char buf[1024], *ptr = buf; +#endif int i; if (!err) return ("Successful return: 0"); @@ -1268,7 +1276,32 @@ mdb_strerror(int err) return mdb_errstr[i]; } +#ifdef _WIN32 + /* These are the C-runtime error codes we use. The comment indicates + * their numeric value, and the Win32 error they would correspond to + * if the error actually came from a Win32 API. A major mess, we should + * have used LMDB-specific error codes for everything. + */ + switch(err) { + case ENOENT: /* 2, FILE_NOT_FOUND */ + case EIO: /* 5, ACCESS_DENIED */ + case ENOMEM: /* 12, INVALID_ACCESS */ + case EACCES: /* 13, INVALID_DATA */ + case EBUSY: /* 16, CURRENT_DIRECTORY */ + case EINVAL: /* 22, BAD_COMMAND */ + case ENOSPC: /* 28, OUT_OF_PAPER */ + return strerror(err); + default: + ; + } + buf[0] = 0; + FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, err, 0, ptr, sizeof(buf), pad); + return ptr; +#else return strerror(err); +#endif } /** assert(3) variant in cursor context */ @@ -2572,7 +2605,7 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret) } if ((txn = calloc(1, size)) == NULL) { - DPRINTF(("calloc: %s", strerror(ErrCode()))); + DPRINTF(("calloc: %s", strerror(errno))); return ENOMEM; } txn->mt_dbs = (MDB_db *) ((char *)txn + tsize); From bda6a60ad4ea32eeae6470bd2bcb7ac5e90931b7 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Wed, 9 Jul 2014 10:22:25 -0700 Subject: [PATCH 14/19] Windows fixes Always set the filesize when opening for writes. Otherwise can't use backups from mdb_copy. --- libraries/liblmdb/mdb.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 4712ac25d6..dc1bb26093 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -3635,7 +3635,7 @@ mdb_env_create(MDB_env **env) } static int ESECT -mdb_env_map(MDB_env *env, void *addr, int newsize) +mdb_env_map(MDB_env *env, void *addr) { MDB_page *p; unsigned int flags = env->me_flags; @@ -3646,6 +3646,7 @@ mdb_env_map(MDB_env *env, void *addr, int newsize) size_t msize; if (flags & MDB_RDONLY) { + /* Don't set explicit map size, use whatever exists */ msize = 0; sizelo = 0; sizehi = 0; @@ -3653,17 +3654,17 @@ mdb_env_map(MDB_env *env, void *addr, int newsize) msize = env->me_mapsize; sizelo = msize & 0xffffffff; sizehi = msize >> 16 >> 16; /* only needed on Win64 */ - } - /* Windows won't create mappings for zero length files. - * Just allocate the maxsize right now. - */ - if (newsize) { + /* Windows won't create mappings for zero length files. + * and won't map more than the file size. + * Just set the maxsize right now. + */ if (SetFilePointer(env->me_fd, sizelo, &sizehi, 0) != (DWORD)sizelo || !SetEndOfFile(env->me_fd) || SetFilePointer(env->me_fd, 0, NULL, 0) != 0) return ErrCode(); } + mh = CreateFileMapping(env->me_fd, NULL, flags & MDB_WRITEMAP ? PAGE_READWRITE : PAGE_READONLY, sizehi, sizelo, NULL); @@ -3747,7 +3748,7 @@ mdb_env_set_mapsize(MDB_env *env, size_t size) munmap(env->me_map, env->me_mapsize); env->me_mapsize = size; old = (env->me_flags & MDB_FIXEDMAP) ? env->me_map : NULL; - rc = mdb_env_map(env, old, 1); + rc = mdb_env_map(env, old); if (rc) return rc; if (change) @@ -3837,7 +3838,7 @@ mdb_env_open2(MDB_env *env) env->me_flags |= MDB_RESIZING; } - rc = mdb_env_map(env, meta.mm_address, newenv || env->me_mapsize != meta.mm_mapsize); + rc = mdb_env_map(env, meta.mm_address); if (rc) return rc; From 02285aca58f5629547263ba09e7dd685dcf6b4b5 Mon Sep 17 00:00:00 2001 From: Hallvard Furuseth Date: Wed, 9 Jul 2014 20:16:33 +0200 Subject: [PATCH 15/19] ITS#7789 Fix resize vs MDB_NOMETASYNC, and a comment. --- libraries/liblmdb/lmdb.h | 2 +- libraries/liblmdb/mdb.c | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h index f63c6719e0..2374ef7cc7 100644 --- a/libraries/liblmdb/lmdb.h +++ b/libraries/liblmdb/lmdb.h @@ -411,7 +411,7 @@ typedef enum MDB_cursor_op { #define MDB_CURSOR_FULL (-30787) /** Page has not enough space - internal error */ #define MDB_PAGE_FULL (-30786) - /** Database contents grew beyond environment mapsize */ + /** Environment mapsize was changed by another process */ #define MDB_MAP_RESIZED (-30785) /** MDB_INCOMPATIBLE: Operation and DB incompatible, or DB flags changed */ #define MDB_INCOMPATIBLE (-30784) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index dc1bb26093..a1199eaa93 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -3323,7 +3323,6 @@ mdb_txn_commit(MDB_txn *txn) #endif if ((rc = mdb_page_flush(txn, 0)) || - (rc = mdb_env_sync(env, 0)) || (rc = mdb_env_write_meta(txn))) goto fail; @@ -3480,7 +3479,7 @@ mdb_env_init_meta(MDB_env *env, MDB_meta *meta) static int mdb_env_write_meta(MDB_txn *txn) { - MDB_env *env; + MDB_env *env = txn->mt_env; MDB_meta meta, metab, *mp; off_t off; int rc, len, toggle; @@ -3492,11 +3491,14 @@ mdb_env_write_meta(MDB_txn *txn) int r2; #endif + /* Sync data and previous metapage before writing a new metapage */ + if ((rc = mdb_env_sync(env, 0)) != MDB_SUCCESS) + return rc; + toggle = txn->mt_txnid & 1; DPRINTF(("writing meta page %d for root page %"Z"u", toggle, txn->mt_dbs[MAIN_DBI].md_root)); - env = txn->mt_env; mp = env->me_metas[toggle]; if (env->me_flags & MDB_WRITEMAP) { From b3dedde6128041a1c30798555ed94f71bcb803ae Mon Sep 17 00:00:00 2001 From: Hallvard Furuseth Date: Wed, 9 Jul 2014 20:17:13 +0200 Subject: [PATCH 16/19] MDB_DEVEL doxygen fix --- libraries/liblmdb/mdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index a1199eaa93..6284edccd1 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -172,7 +172,7 @@ * @{ */ -/* Features under development */ + /** Features under development */ #ifndef MDB_DEVEL #define MDB_DEVEL 0 #endif From 2825ad64c2ee0dbba56b379d227c400be772d88c Mon Sep 17 00:00:00 2001 From: Hallvard Furuseth Date: Wed, 9 Jul 2014 20:18:08 +0200 Subject: [PATCH 17/19] MDB_MAXKEYSIZE = 0 (dynamic value) when MDB_DEVEL --- libraries/liblmdb/mdb.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 6284edccd1..2b41c2e2dd 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -427,13 +427,14 @@ static txnid_t mdb_debug_start; * Define this as 0 to compute the max from the page size. 511 * is default for backwards compat: liblmdb <= 0.9.10 can break * when modifying a DB with keys/dupsort data bigger than its max. + * #MDB_DEVEL sets the default to 0. * * Data items in an #MDB_DUPSORT database are also limited to * this size, since they're actually keys of a sub-DB. Keys and * #MDB_DUPSORT data items must fit on a node in a regular page. */ #ifndef MDB_MAXKEYSIZE -#define MDB_MAXKEYSIZE 511 +#define MDB_MAXKEYSIZE ((MDB_DEVEL) ? 0 : 511) #endif /** The maximum size of a key we can write to the environment. */ From 7f038d0f15bec57b4c07aa3f31cd5564c88a1897 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Wed, 9 Jul 2014 12:05:37 -0700 Subject: [PATCH 18/19] Zero-init copy2 writebuf --- libraries/liblmdb/mdb.c | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 2b41c2e2dd..5e7716151d 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -8445,6 +8445,7 @@ mdb_env_copyfd1(MDB_env *env, HANDLE fd) if (rc) return rc; #endif + memset(my.mc_wbuf[0], 0, MDB_WBUF*2); my.mc_wbuf[1] = my.mc_wbuf[0] + MDB_WBUF; my.mc_wlen[0] = 0; my.mc_wlen[1] = 0; From 886ee41d55a410ebadf81cbba3082fdffeed18fd Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Wed, 9 Jul 2014 15:03:54 -0700 Subject: [PATCH 19/19] More for cursor EOF Must also set in sub-cursor if deleting entire subDB --- libraries/liblmdb/mdb.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 5e7716151d..f8c94e3100 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -7659,6 +7659,8 @@ mdb_cursor_del0(MDB_cursor *mc) m3->mc_flags |= C_DEL; if (m3->mc_ki[mc->mc_top] > ki) m3->mc_ki[mc->mc_top]--; + else if (mc->mc_db->md_flags & MDB_DUPSORT) + m3->mc_xcursor->mx_cursor.mc_flags |= C_EOF; } if (m3->mc_ki[mc->mc_top] >= nkeys) { rc = mdb_cursor_sibling(m3, 1);