From 54f00bd01f37e252cf01c643036e8d2d45e2e0ef Mon Sep 17 00:00:00 2001 From: Quanah Gibson-Mount Date: Mon, 24 Jun 2024 16:56:14 +0000 Subject: [PATCH 01/15] Return to engineering --- libraries/liblmdb/CHANGES | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libraries/liblmdb/CHANGES b/libraries/liblmdb/CHANGES index 20babf3db5..d138bbb932 100644 --- a/libraries/liblmdb/CHANGES +++ b/libraries/liblmdb/CHANGES @@ -1,5 +1,7 @@ LMDB 0.9 Change Log +LMDB 0.9.34 Engineering + LMDB 0.9.33 Release (2024/05/21) ITS#9037 mdb_page_search: fix error code when DBI record is missing ITS#10198 For win32, stop passing ignored parameter From b84c35053b5a0068c5215a2c02628b17969868e3 Mon Sep 17 00:00:00 2001 From: Zach Vonler Date: Mon, 3 Jun 2024 12:39:02 -0700 Subject: [PATCH 02/15] ITS#10222 LMDB: Updated mdb_dump man page The -a option to mdb_load makes the previous text obsolete. --- libraries/liblmdb/mdb_dump.1 | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/libraries/liblmdb/mdb_dump.1 b/libraries/liblmdb/mdb_dump.1 index 5f2d771b58..9674c29f7b 100644 --- a/libraries/liblmdb/mdb_dump.1 +++ b/libraries/liblmdb/mdb_dump.1 @@ -61,14 +61,14 @@ Exit status is zero if no errors occur. Errors result in a non-zero exit status and a diagnostic message being written to standard error. -Dumping and reloading databases that use user-defined comparison functions -will result in new databases that use the default comparison functions. -\fBIn this case it is quite likely that the reloaded database will be -damaged beyond repair permitting neither record storage nor retrieval.\fP +Dumping databases that use user-defined comparison functions will output +records with the ordering imposed by those comparison functions. If +.B mdb_load +is invoked without including the +.B -a +option when reloading those records, the new databases will likely be +damaged beyond repair, permitting neither record storage nor retrieval. -The only available workaround is to modify the source for the -.BR mdb_load (1) -utility to load the database using the correct comparison functions. .SH "SEE ALSO" .BR mdb_load (1) .SH AUTHOR From c6a49d7963799d92e1b0f2133696ef9805070942 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Mon, 3 Jun 2024 21:14:46 +0100 Subject: [PATCH 03/15] LMDB: tweak mdb_load.1 manpage Add missing -a option to Synopsis --- libraries/liblmdb/mdb_load.1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libraries/liblmdb/mdb_load.1 b/libraries/liblmdb/mdb_load.1 index 78439a14da..6f1c8a7196 100644 --- a/libraries/liblmdb/mdb_load.1 +++ b/libraries/liblmdb/mdb_load.1 @@ -8,6 +8,8 @@ mdb_load \- LMDB environment import tool [\c .BR \-V ] [\c +.BR \-a ] +[\c .BI \-f \ file\fR] [\c .BR \-n ] From f6d015dd84002c4e0bfb739a5b99769cc6137887 Mon Sep 17 00:00:00 2001 From: Quanah Gibson-Mount Date: Fri, 28 Jun 2024 15:55:18 +0000 Subject: [PATCH 04/15] ITS#10222 --- libraries/liblmdb/CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/liblmdb/CHANGES b/libraries/liblmdb/CHANGES index d138bbb932..1cbab56def 100644 --- a/libraries/liblmdb/CHANGES +++ b/libraries/liblmdb/CHANGES @@ -1,6 +1,7 @@ LMDB 0.9 Change Log LMDB 0.9.34 Engineering + ITS#10222 - Update mdb_dump(1) and mdb_load(1) man pages for append (-a) option LMDB 0.9.33 Release (2024/05/21) ITS#9037 mdb_page_search: fix error code when DBI record is missing From c5f0ac271cd1b5e74fca89d9a9ffb62f96b5b873 Mon Sep 17 00:00:00 2001 From: Gary Wicker Date: Fri, 25 Oct 2024 01:05:00 +0000 Subject: [PATCH 05/15] ITS#10275 mdb_load: add -Q option to use NOSYNC for faster loading --- libraries/liblmdb/mdb_load.1 | 5 +++++ libraries/liblmdb/mdb_load.c | 17 ++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/libraries/liblmdb/mdb_load.1 b/libraries/liblmdb/mdb_load.1 index 6f1c8a7196..741555f87c 100644 --- a/libraries/liblmdb/mdb_load.1 +++ b/libraries/liblmdb/mdb_load.1 @@ -18,6 +18,8 @@ mdb_load \- LMDB environment import tool [\c .BR \-N ] [\c +.BR \-Q ] +[\c .BR \-T ] .BR \ envpath .SH DESCRIPTION @@ -58,6 +60,9 @@ Load a specific subdatabase. If no database is specified, data is loaded into th .BR \-N Don't overwrite existing records when loading into an already existing database; just skip them. .TP +.BR \-Q +Quick mode, uses MDB_NOSYNC for faster loading. Forces sync with mdb_env_sync() before exiting. +.TP .BR \-T Load data from simple text files. The input must be paired lines of text, where the first line of the pair is the key item, and the second line of the pair is its corresponding diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c index cba6c06003..d266b4f6e1 100644 --- a/libraries/liblmdb/mdb_load.c +++ b/libraries/liblmdb/mdb_load.c @@ -311,10 +311,11 @@ int main(int argc, char *argv[]) * -n: use NOSUBDIR flag on env_open * -s: load into named subDB * -N: use NOOVERWRITE on puts + * -Q: quick mode using NOSYNC * -T: read plaintext * -V: print version and exit */ - while ((i = getopt(argc, argv, "af:ns:NTV")) != EOF) { + while ((i = getopt(argc, argv, "af:ns:NQTV")) != EOF) { switch(i) { case 'V': printf("%s\n", MDB_VERSION_STRING); @@ -339,6 +340,9 @@ int main(int argc, char *argv[]) case 'N': putflags = MDB_NOOVERWRITE|MDB_NODUPDATA; break; + case 'Q': + envflags |= MDB_NOSYNC; + break; case 'T': mode |= NOHDR | PRINT; break; @@ -403,9 +407,9 @@ int main(int argc, char *argv[]) goto env_close; } - rc = mdb_open(txn, subname, flags|MDB_CREATE, &dbi); + rc = mdb_dbi_open(txn, subname, flags|MDB_CREATE, &dbi); if (rc) { - fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc)); + fprintf(stderr, "mdb_dbi_open failed, error %d %s\n", rc, mdb_strerror(rc)); goto txn_abort; } prevk.mv_size = 0; @@ -486,6 +490,13 @@ int main(int argc, char *argv[]) prog, lineno, mdb_strerror(rc)); goto env_close; } + if (envflags & MDB_NOSYNC) { + rc = mdb_env_sync(env, 1); + if (rc) { + fprintf(stderr, "mdb_env_sync failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; + } + } mdb_dbi_close(env, dbi); } From 790ed0e9119ce9d0a7dc4fe14a6a70b5396af315 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Sat, 26 Oct 2024 21:27:01 +0100 Subject: [PATCH 06/15] ITS#10275 mdb_load: add -Q option to use NOSYNC for faster loading --- libraries/liblmdb/CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/liblmdb/CHANGES b/libraries/liblmdb/CHANGES index 1cbab56def..f3a8a551c2 100644 --- a/libraries/liblmdb/CHANGES +++ b/libraries/liblmdb/CHANGES @@ -2,6 +2,7 @@ LMDB 0.9 Change Log LMDB 0.9.34 Engineering ITS#10222 - Update mdb_dump(1) and mdb_load(1) man pages for append (-a) option + ITS#10275 - mdb_load: add -Q option to use NOSYNC LMDB 0.9.33 Release (2024/05/21) ITS#9037 mdb_page_search: fix error code when DBI record is missing From f6b3c3d407862502d65f3acffa5587a3a07dac57 Mon Sep 17 00:00:00 2001 From: kero Date: Thu, 9 Jan 2025 14:56:00 +0000 Subject: [PATCH 07/15] ITS#10296 lmdb: fix fdatasync on MacOS Patch from kero, with corrections by hyc --- libraries/liblmdb/mdb.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 668f9660f2..25307144ed 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -128,7 +128,11 @@ typedef SSIZE_T ssize_t; # define MDB_USE_ROBUST 1 #elif defined(__APPLE__) || defined (BSD) || defined(__FreeBSD_kernel__) # define MDB_USE_POSIX_SEM 1 -# define MDB_FDATASYNC fsync +# if defined(__APPLE__) +# define MDB_FDATASYNC fcntl(fd, F_FULLSYNC) +# else +# define MDB_FDATASYNC fsync +# endif #elif defined(ANDROID) # define MDB_FDATASYNC fsync #endif @@ -2573,7 +2577,7 @@ mdb_env_sync(MDB_env *env, int force) ? MS_ASYNC : MS_SYNC; if (MDB_MSYNC(env->me_map, env->me_mapsize, flags)) rc = ErrCode(); -#ifdef _WIN32 +#if defined(_WIN32) || defined(__APPLE__) else if (flags == MS_SYNC && MDB_FDATASYNC(env->me_fd)) rc = ErrCode(); #endif From 96bffc92026e2cb17aa1b9f4c39e80890e194e92 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Tue, 28 Jan 2025 17:29:56 +0000 Subject: [PATCH 08/15] ITS#10296 fix fdatasync on MacOS --- libraries/liblmdb/CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/liblmdb/CHANGES b/libraries/liblmdb/CHANGES index f3a8a551c2..f86ce522c2 100644 --- a/libraries/liblmdb/CHANGES +++ b/libraries/liblmdb/CHANGES @@ -3,6 +3,7 @@ LMDB 0.9 Change Log LMDB 0.9.34 Engineering ITS#10222 - Update mdb_dump(1) and mdb_load(1) man pages for append (-a) option ITS#10275 - mdb_load: add -Q option to use NOSYNC + ITS#10296 - fix fdatasync on MacOS LMDB 0.9.33 Release (2024/05/21) ITS#9037 mdb_page_search: fix error code when DBI record is missing From ac89f42433bfcc8ffb62665d2364ce8b072975d5 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Wed, 19 Feb 2025 09:15:27 +0000 Subject: [PATCH 09/15] ITS#10296 lmdb: fix typo in prev commit --- libraries/liblmdb/mdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 25307144ed..fcc66aae52 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -129,7 +129,7 @@ typedef SSIZE_T ssize_t; #elif defined(__APPLE__) || defined (BSD) || defined(__FreeBSD_kernel__) # define MDB_USE_POSIX_SEM 1 # if defined(__APPLE__) -# define MDB_FDATASYNC fcntl(fd, F_FULLSYNC) +# define MDB_FDATASYNC fcntl(fd, F_FULLFSYNC) # else # define MDB_FDATASYNC fsync # endif From ce339e82bd3e711e3fd675e3e6606c2516409160 Mon Sep 17 00:00:00 2001 From: jinyaoguo Date: Tue, 27 May 2025 21:47:48 -0400 Subject: [PATCH 10/15] ITS#10342 lmdb: fix potential memleak in child txn_begin --- libraries/liblmdb/mdb.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index fcc66aae52..9fb142d23d 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -2974,8 +2974,10 @@ renew: rc = mdb_txn_renew0(txn); } if (rc) { - if (txn != env->me_txn0) + if (txn != env->me_txn0) { + free(txn->mt_u.dirty_list); free(txn); + } } else { txn->mt_flags |= flags; /* could not change txn=me_txn0 earlier */ *ret = txn; From 529442b4cb688328a99fd7becb4c49465ae4ada6 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Tue, 10 Jun 2025 20:43:44 +0100 Subject: [PATCH 11/15] ITS#10342 --- libraries/liblmdb/CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/liblmdb/CHANGES b/libraries/liblmdb/CHANGES index f86ce522c2..47b01479b4 100644 --- a/libraries/liblmdb/CHANGES +++ b/libraries/liblmdb/CHANGES @@ -4,6 +4,7 @@ LMDB 0.9.34 Engineering ITS#10222 - Update mdb_dump(1) and mdb_load(1) man pages for append (-a) option ITS#10275 - mdb_load: add -Q option to use NOSYNC ITS#10296 - fix fdatasync on MacOS + ITS#10342 - fix memleak in mdb_txn_begin for nested txns LMDB 0.9.33 Release (2024/05/21) ITS#9037 mdb_page_search: fix error code when DBI record is missing From 7f8f52e0e225d13e85f49a18cde582de916fa611 Mon Sep 17 00:00:00 2001 From: Mike Moritz Date: Wed, 28 May 2025 23:11:38 +0000 Subject: [PATCH 12/15] ITS#10346 lmdb: fix compacting copy with large values --- libraries/liblmdb/mdb.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 9fb142d23d..7a52945c3b 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -9176,8 +9176,8 @@ typedef struct mdb_copy { pthread_cond_t mc_cond; /**< Condition variable for #mc_new */ char *mc_wbuf[2]; char *mc_over[2]; - int mc_wlen[2]; - int mc_olen[2]; + size_t mc_wlen[2]; + size_t mc_olen[2]; pgno_t mc_next_pgno; HANDLE mc_fd; int mc_toggle; /**< Buffer number in provider */ @@ -9194,7 +9194,8 @@ mdb_env_copythr(void *arg) { mdb_copy *my = arg; char *ptr; - int toggle = 0, wsize, rc; + int toggle = 0, rc; + size_t wsize; #ifdef _WIN32 DWORD len; #define DO_WRITE(rc, fd, ptr, w2, len) rc = WriteFile(fd, ptr, w2, &len, NULL) From c85bcff9c59df7c4669faa814d7925ad00a71d77 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Tue, 10 Jun 2025 20:57:48 +0100 Subject: [PATCH 13/15] ITS#10346 --- libraries/liblmdb/CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/liblmdb/CHANGES b/libraries/liblmdb/CHANGES index 47b01479b4..1d949b1f20 100644 --- a/libraries/liblmdb/CHANGES +++ b/libraries/liblmdb/CHANGES @@ -5,6 +5,7 @@ LMDB 0.9.34 Engineering ITS#10275 - mdb_load: add -Q option to use NOSYNC ITS#10296 - fix fdatasync on MacOS ITS#10342 - fix memleak in mdb_txn_begin for nested txns + ITS#10342 - fix mdb_env_copy2 with values > (2GB-16) LMDB 0.9.33 Release (2024/05/21) ITS#9037 mdb_page_search: fix error code when DBI record is missing From 7c7988003b29514464d1a9c607e6991c25c8b88d Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Thu, 12 Jun 2025 15:16:00 +0100 Subject: [PATCH 14/15] ITS#10355 lmdb mplay: don't assign to stdin/stdout --- libraries/liblmdb/mplay.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/libraries/liblmdb/mplay.c b/libraries/liblmdb/mplay.c index 0fef74f67e..4673036774 100644 --- a/libraries/liblmdb/mplay.c +++ b/libraries/liblmdb/mplay.c @@ -483,12 +483,10 @@ static pidpair *addpid(int tpid) pipe(fdin); if ((pid = fork()) == 0) { /* child */ - fclose(stdin); - fclose(stdout); + fflush(stdin); + fflush(stdout); dup2(fdout[0], 0); dup2(fdin[1], 1); - stdin = fdopen(0, "r"); - stdout = fdopen(1, "w"); child(); return 0; /* NOTREACHED */ } else { From f7c1754e36c6790d1209966bafc0f56e4e051e97 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Thu, 12 Jun 2025 15:18:48 +0100 Subject: [PATCH 15/15] ITS#10355 --- libraries/liblmdb/CHANGES | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/liblmdb/CHANGES b/libraries/liblmdb/CHANGES index 1d949b1f20..857a08e14c 100644 --- a/libraries/liblmdb/CHANGES +++ b/libraries/liblmdb/CHANGES @@ -5,7 +5,8 @@ LMDB 0.9.34 Engineering ITS#10275 - mdb_load: add -Q option to use NOSYNC ITS#10296 - fix fdatasync on MacOS ITS#10342 - fix memleak in mdb_txn_begin for nested txns - ITS#10342 - fix mdb_env_copy2 with values > (2GB-16) + ITS#10346 - fix mdb_env_copy2 with values > (2GB-16) + ITS#10355 - fix mplay build on musl LMDB 0.9.33 Release (2024/05/21) ITS#9037 mdb_page_search: fix error code when DBI record is missing