From 6f53771687751673b6c79cdbfd5a2de8d0620d30 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Sun, 4 Oct 2015 03:02:04 +0100 Subject: [PATCH 1/7] ITS#8258 --- libraries/liblmdb/CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/liblmdb/CHANGES b/libraries/liblmdb/CHANGES index a6304660bb..03bacb404d 100644 --- a/libraries/liblmdb/CHANGES +++ b/libraries/liblmdb/CHANGES @@ -4,6 +4,7 @@ LMDB 0.9.17 Release Engineering Fix ITS#7377 catch calloc failure Fix ITS#8237 regression from ITS#7589 Fix ITS#8221 MDB_PAGE_FULL on delete/rebalance + Fix ITS#8258 rebalance/split assert Build Create install dirs if needed (ITS#8256) From 9ed1e574401b2b7224119aae86a6d9e8a9786849 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Tue, 6 Oct 2015 07:57:14 +0100 Subject: [PATCH 2/7] ITS#8264 fix cursor_del cursor tracking Some destination fixups need to happen immediately after nodes are moved, before rebalancing --- libraries/liblmdb/mdb.c | 72 +++++++++++++++++++++++++---------------- 1 file changed, 45 insertions(+), 27 deletions(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index fcfdd05275..47ba402f3c 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -7440,8 +7440,22 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst) /* Adjust other cursors pointing to mp */ MDB_cursor *m2, *m3; MDB_dbi dbi = csrc->mc_dbi; - MDB_page *mp = csrc->mc_pg[csrc->mc_top]; + MDB_page *mp; + mp = cdst->mc_pg[csrc->mc_top]; + for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (csrc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == cdst) continue; + if (m3->mc_pg[csrc->mc_top] == mp && m3->mc_ki[csrc->mc_top] >= + cdst->mc_ki[csrc->mc_top]) { + m3->mc_ki[csrc->mc_top]++; + } + } + + mp = csrc->mc_pg[csrc->mc_top]; for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { if (csrc->mc_flags & C_SUB) m3 = &m2->mc_xcursor->mx_cursor; @@ -7877,37 +7891,16 @@ mdb_cursor_del0(MDB_cursor *mc) MDB_page *mp; indx_t ki; unsigned int nkeys; + MDB_cursor *m2, *m3; + MDB_dbi dbi = mc->mc_dbi; ki = mc->mc_ki[mc->mc_top]; + mp = mc->mc_pg[mc->mc_top]; mdb_node_del(mc, mc->mc_db->md_pad); mc->mc_db->md_entries--; - rc = mdb_rebalance(mc); - - if (rc == MDB_SUCCESS) { - MDB_cursor *m2, *m3; - MDB_dbi dbi = mc->mc_dbi; - - /* DB is totally empty now, just bail out. - * Other cursors adjustments were already done - * by mdb_rebalance and aren't needed here. - */ - if (!mc->mc_snum) - return rc; - - mp = mc->mc_pg[mc->mc_top]; - nkeys = NUMKEYS(mp); - - /* if mc points past last node in page, find next sibling */ - if (mc->mc_ki[mc->mc_top] >= nkeys) { - rc = mdb_cursor_sibling(mc, 1); - if (rc == MDB_NOTFOUND) { - mc->mc_flags |= C_EOF; - rc = MDB_SUCCESS; - } - } - + { /* Adjust other cursors pointing to mp */ - for (m2 = mc->mc_txn->mt_cursors[dbi]; !rc && m2; m2=m2->mc_next) { + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; if (! (m2->mc_flags & m3->mc_flags & C_INITIALIZED)) continue; @@ -7921,6 +7914,31 @@ mdb_cursor_del0(MDB_cursor *mc) else if (mc->mc_db->md_flags & MDB_DUPSORT) m3->mc_xcursor->mx_cursor.mc_flags |= C_EOF; } + } + } + } + rc = mdb_rebalance(mc); + + if (rc == MDB_SUCCESS) { + /* DB is totally empty now, just bail out. + * Other cursors adjustments were already done + * by mdb_rebalance and aren't needed here. + */ + if (!mc->mc_snum) + return rc; + + mp = mc->mc_pg[mc->mc_top]; + nkeys = NUMKEYS(mp); + + /* Adjust other cursors pointing to mp */ + for (m2 = mc->mc_txn->mt_cursors[dbi]; !rc && m2; m2=m2->mc_next) { + m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; + if (! (m2->mc_flags & m3->mc_flags & C_INITIALIZED)) + continue; + if (m3->mc_snum < mc->mc_snum) + continue; + if (m3->mc_pg[mc->mc_top] == mp) { + /* if m3 points past last node in page, find next sibling */ if (m3->mc_ki[mc->mc_top] >= nkeys) { rc = mdb_cursor_sibling(m3, 1); if (rc == MDB_NOTFOUND) { From 7e3c53282323aa83ebd332798bdd4350c9b71e18 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Wed, 7 Oct 2015 06:30:58 +0100 Subject: [PATCH 3/7] ITS#8263 fix cursor tracking in cursor_put --- libraries/liblmdb/mdb.c | 77 +++++++++++++++++++++++++++++++++-------- 1 file changed, 63 insertions(+), 14 deletions(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 47ba402f3c..e0884e5a6c 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -1242,6 +1242,7 @@ static int mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data); static void mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx); static void mdb_xcursor_init0(MDB_cursor *mc); static void mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node); +static void mdb_xcursor_init2(MDB_cursor *mc, MDB_node *node); static int mdb_drop0(MDB_cursor *mc, int subs); static void mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi); @@ -6554,23 +6555,23 @@ put_sub: rc = mdb_cursor_put(&mc->mc_xcursor->mx_cursor, &dkey, &xdata, xflags); if (rc) goto bad_sub; - { - /* Adjust other cursors pointing to mp */ - MDB_cursor *m2; - unsigned i = mc->mc_top; - MDB_page *mp = mc->mc_pg[i]; - - for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { - if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; - if (!(m2->mc_flags & C_INITIALIZED)) continue; - if (m2->mc_pg[i] == mp && m2->mc_ki[i] == mc->mc_ki[i]) { - mdb_xcursor_init1(m2, leaf); - } - } - } /* we've done our job */ dkey.mv_size = 0; } + { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2; + unsigned i = mc->mc_top; + MDB_page *mp = mc->mc_pg[i]; + + for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { + if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; + if (!(m2->mc_flags & C_INITIALIZED)) continue; + if (m2->mc_pg[i] == mp && m2->mc_ki[i] == mc->mc_ki[i]) { + mdb_xcursor_init2(m2, leaf); + } + } + } ecount = mc->mc_xcursor->mx_db.md_entries; if (flags & MDB_APPENDDUP) xflags |= MDB_APPEND; @@ -7122,6 +7123,54 @@ mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node) #endif } + +/** Fixup a sorted-dups cursor due to underlying update. + * Sets up some fields that depend on the data from the main cursor. + * Almost the same as init1, but skips initialization steps if the + * xcursor had already been used. + * @param[in] mc The main cursor whose sorted-dups cursor is to be fixed up. + * @param[in] node The data containing the #MDB_db record for the + * sorted-dup database. + */ +static void +mdb_xcursor_init2(MDB_cursor *mc, MDB_node *node) +{ + MDB_xcursor *mx = mc->mc_xcursor; + + if (node->mn_flags & F_SUBDATA) { + memcpy(&mx->mx_db, NODEDATA(node), sizeof(MDB_db)); + mdb_page_get(mc->mc_txn,mx->mx_db.md_root,&mx->mx_cursor.mc_pg[0],NULL); + } else { + MDB_page *fp = NODEDATA(node); + mx->mx_db.md_entries = NUMKEYS(fp); + COPY_PGNO(mx->mx_db.md_root, fp->mp_pgno); + mx->mx_cursor.mc_pg[0] = fp; + } + if (!(mx->mx_cursor.mc_flags & C_INITIALIZED)) { + mx->mx_cursor.mc_snum = 1; + mx->mx_cursor.mc_top = 0; + mx->mx_cursor.mc_flags |= C_INITIALIZED; + mx->mx_cursor.mc_ki[0] = 0; + if (!(node->mn_flags & F_SUBDATA)) { + mx->mx_db.md_pad = 0; + mx->mx_db.md_flags = 0; + mx->mx_db.md_depth = 1; + mx->mx_db.md_branch_pages = 0; + mx->mx_db.md_leaf_pages = 1; + mx->mx_db.md_overflow_pages = 0; + if (mc->mc_db->md_flags & MDB_DUPFIXED) { + mx->mx_db.md_flags = MDB_DUPFIXED; + mx->mx_db.md_pad = mx->mx_cursor.mc_pg[0]->mp_pad; + if (mc->mc_db->md_flags & MDB_INTEGERDUP) + mx->mx_db.md_flags |= MDB_INTEGERKEY; + } + } + } + DPRINTF(("Sub-db -%u root page %"Z"u", mx->mx_cursor.mc_dbi, + mx->mx_db.md_root)); + mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DIRTY; /* DB_DIRTY guides mdb_cursor_touch */ +} + /** Initialize a cursor for a given transaction and database. */ static void mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx) From 448b4c815a3f6c986352f7f7c35afc10a24887b1 Mon Sep 17 00:00:00 2001 From: Hallvard Furuseth Date: Sun, 11 Oct 2015 21:40:49 +0100 Subject: [PATCH 4/7] ITS#8263 streamline prev patch --- libraries/liblmdb/mdb.c | 55 +++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 33 deletions(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index e0884e5a6c..03c4429169 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -1242,7 +1242,7 @@ static int mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data); static void mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx); static void mdb_xcursor_init0(MDB_cursor *mc); static void mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node); -static void mdb_xcursor_init2(MDB_cursor *mc, MDB_node *node); +static void mdb_xcursor_init2(MDB_cursor *mc, MDB_xcursor *src_mx, int force); static int mdb_drop0(MDB_cursor *mc, int subs); static void mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi); @@ -6111,7 +6111,7 @@ mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, enum { MDB_NO_ROOT = MDB_LAST_ERRCODE+10 }; /* internal code */ MDB_env *env; MDB_node *leaf = NULL; - MDB_page *fp, *mp; + MDB_page *fp, *mp, *sub_root = NULL; uint16_t fp_flags; MDB_val xdata, *rdata, dkey, olddata; MDB_db dummy; @@ -6391,6 +6391,7 @@ prep_subDB: offset = env->me_psize - olddata.mv_size; flags |= F_DUPDATA|F_SUBDATA; dummy.md_root = mp->mp_pgno; + sub_root = mp; } if (mp != fp) { mp->mp_flags = fp_flags | P_DIRTY; @@ -6537,7 +6538,7 @@ new_sub: * DB are all zero size. */ if (do_sub) { - int xflags; + int xflags, new_dupdata; size_t ecount; put_sub: xdata.mv_size = 0; @@ -6550,6 +6551,9 @@ put_sub: xflags = (flags & MDB_NODUPDATA) ? MDB_NOOVERWRITE|MDB_NOSPILL : MDB_NOSPILL; } + if (sub_root) + mc->mc_xcursor->mx_cursor.mc_pg[0] = sub_root; + new_dupdata = (int)dkey.mv_size; /* converted, write the original data first */ if (dkey.mv_size) { rc = mdb_cursor_put(&mc->mc_xcursor->mx_cursor, &dkey, &xdata, xflags); @@ -6558,9 +6562,10 @@ put_sub: /* we've done our job */ dkey.mv_size = 0; } - { + if (!(leaf->mn_flags & F_SUBDATA) || sub_root) { /* Adjust other cursors pointing to mp */ MDB_cursor *m2; + MDB_xcursor *mx = mc->mc_xcursor; unsigned i = mc->mc_top; MDB_page *mp = mc->mc_pg[i]; @@ -6568,7 +6573,7 @@ put_sub: if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; if (!(m2->mc_flags & C_INITIALIZED)) continue; if (m2->mc_pg[i] == mp && m2->mc_ki[i] == mc->mc_ki[i]) { - mdb_xcursor_init2(m2, leaf); + mdb_xcursor_init2(m2, mx, new_dupdata); } } } @@ -7129,46 +7134,30 @@ mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node) * Almost the same as init1, but skips initialization steps if the * xcursor had already been used. * @param[in] mc The main cursor whose sorted-dups cursor is to be fixed up. - * @param[in] node The data containing the #MDB_db record for the - * sorted-dup database. + * @param[in] src_mx The xcursor of an up-to-date cursor. + * @param[in] new_dupdata True if converting from a non-#F_DUPDATA item. */ static void -mdb_xcursor_init2(MDB_cursor *mc, MDB_node *node) +mdb_xcursor_init2(MDB_cursor *mc, MDB_xcursor *src_mx, int new_dupdata) { MDB_xcursor *mx = mc->mc_xcursor; - if (node->mn_flags & F_SUBDATA) { - memcpy(&mx->mx_db, NODEDATA(node), sizeof(MDB_db)); - mdb_page_get(mc->mc_txn,mx->mx_db.md_root,&mx->mx_cursor.mc_pg[0],NULL); - } else { - MDB_page *fp = NODEDATA(node); - mx->mx_db.md_entries = NUMKEYS(fp); - COPY_PGNO(mx->mx_db.md_root, fp->mp_pgno); - mx->mx_cursor.mc_pg[0] = fp; - } - if (!(mx->mx_cursor.mc_flags & C_INITIALIZED)) { + if (new_dupdata) { mx->mx_cursor.mc_snum = 1; mx->mx_cursor.mc_top = 0; mx->mx_cursor.mc_flags |= C_INITIALIZED; mx->mx_cursor.mc_ki[0] = 0; - if (!(node->mn_flags & F_SUBDATA)) { - mx->mx_db.md_pad = 0; - mx->mx_db.md_flags = 0; - mx->mx_db.md_depth = 1; - mx->mx_db.md_branch_pages = 0; - mx->mx_db.md_leaf_pages = 1; - mx->mx_db.md_overflow_pages = 0; - if (mc->mc_db->md_flags & MDB_DUPFIXED) { - mx->mx_db.md_flags = MDB_DUPFIXED; - mx->mx_db.md_pad = mx->mx_cursor.mc_pg[0]->mp_pad; - if (mc->mc_db->md_flags & MDB_INTEGERDUP) - mx->mx_db.md_flags |= MDB_INTEGERKEY; - } - } + mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DIRTY; /* DB_DIRTY guides mdb_cursor_touch */ +#if UINT_MAX < SIZE_MAX + mx->mx_dbx.md_cmp = src_mx->mx_dbx.md_cmp; +#endif + } else if (!(mx->mx_cursor.mc_flags & C_INITIALIZED)) { + return; } + mx->mx_db = src_mx->mx_db; + mx->mx_cursor.mc_pg[0] = src_mx->mx_cursor.mc_pg[0]; DPRINTF(("Sub-db -%u root page %"Z"u", mx->mx_cursor.mc_dbi, mx->mx_db.md_root)); - mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DIRTY; /* DB_DIRTY guides mdb_cursor_touch */ } /** Initialize a cursor for a given transaction and database. */ From 24add6589b62751b5a0f560d6199e67d7706480b Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Mon, 12 Oct 2015 00:08:41 +0100 Subject: [PATCH 5/7] ITS#7771 fix cursor tracking on fake pages node_del shifts nodes around, cursors pointing at fake pages need to have their mc_pg[0] corrected. --- libraries/liblmdb/mdb.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 03c4429169..1dc7152c7f 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -6572,8 +6572,14 @@ put_sub: for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; if (!(m2->mc_flags & C_INITIALIZED)) continue; - if (m2->mc_pg[i] == mp && m2->mc_ki[i] == mc->mc_ki[i]) { - mdb_xcursor_init2(m2, mx, new_dupdata); + if (m2->mc_pg[i] == mp) { + if (m2->mc_ki[i] == mc->mc_ki[i]) { + mdb_xcursor_init2(m2, mx, new_dupdata); + } else if (!insert_key) { + MDB_node *n2 = NODEPTR(mp, m2->mc_ki[i]); + if (!(n2->mn_flags & F_SUBDATA)) + m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(n2); + } } } } @@ -6671,12 +6677,19 @@ mdb_cursor_del(MDB_cursor *mc, unsigned int flags) mdb_node_shrink(mp, mc->mc_ki[mc->mc_top]); leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); mc->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); - /* fix other sub-DB cursors pointed at this fake page */ + /* fix other sub-DB cursors pointed at fake pages on this page */ for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; - if (m2->mc_pg[mc->mc_top] == mp && - m2->mc_ki[mc->mc_top] == mc->mc_ki[mc->mc_top]) - m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); + if (!(m2->mc_flags & C_INITIALIZED)) continue; + if (m2->mc_pg[mc->mc_top] == mp) { + if (m2->mc_ki[mc->mc_top] == mc->mc_ki[mc->mc_top]) { + m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); + } else { + MDB_node *n2 = NODEPTR(mp, m2->mc_ki[mc->mc_top]); + if (!(n2->mn_flags & F_SUBDATA)) + m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(n2); + } + } } } mc->mc_db->md_entries--; From 52446f93e7bc7c075b47d2e584a26c978f21f406 Mon Sep 17 00:00:00 2001 From: Ignacio Casal Quinteiro Date: Thu, 15 Oct 2015 13:22:25 -0700 Subject: [PATCH 6/7] ITS#8270 win32: fix conversion error with MSVC --- libraries/liblmdb/mdb.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 1dc7152c7f..828e43205d 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -185,6 +185,12 @@ extern int cacheflush(char *addr, int nbytes, int cache); #define ESECT #endif +#ifdef _MSC_VER +#define CALL_CONV WINAPI +#else +#define CALL_CONV +#endif + /** @defgroup internal LMDB Internals * @{ */ @@ -8509,7 +8515,7 @@ typedef struct mdb_copy { } mdb_copy; /** Dedicated writer thread for compacting copy. */ -static THREAD_RET ESECT +static THREAD_RET ESECT CALL_CONV mdb_env_copythr(void *arg) { mdb_copy *my = arg; From fc3514152f65cf6aae839ada8c9a0c8dc3c7beda Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Sun, 25 Oct 2015 08:13:25 +0000 Subject: [PATCH 7/7] ITS#8263, 8264, 7771, 8270 --- libraries/liblmdb/CHANGES | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libraries/liblmdb/CHANGES b/libraries/liblmdb/CHANGES index 03bacb404d..f99815db90 100644 --- a/libraries/liblmdb/CHANGES +++ b/libraries/liblmdb/CHANGES @@ -5,8 +5,12 @@ LMDB 0.9.17 Release Engineering Fix ITS#8237 regression from ITS#7589 Fix ITS#8221 MDB_PAGE_FULL on delete/rebalance Fix ITS#8258 rebalance/split assert + Fix ITS#8264 cursor_del cursor tracking + Fix ITS#8263 cursor_put cursor tracking + Fix ITS#7771 fakepage cursor tracking Build Create install dirs if needed (ITS#8256) + Fix ThreadProc decl on Win32/MSVC (ITS#8270) LMDB 0.9.16 Release (2015/08/14) Fix cursor EOF bug (ITS#8190)