From 3e83b5b27bcd76a5ae086573516f9b76bdc6e2da Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Fri, 20 Jun 2014 05:33:31 -0700 Subject: [PATCH 01/16] Add mdb_dump, update copyrights --- libraries/liblmdb/Makefile | 5 +- libraries/liblmdb/lmdb.h | 6 +- libraries/liblmdb/mdb.c | 2 +- libraries/liblmdb/mdb_copy.1 | 10 +- libraries/liblmdb/mdb_copy.c | 7 +- libraries/liblmdb/mdb_dump.1 | 70 +++++++++ libraries/liblmdb/mdb_dump.c | 276 +++++++++++++++++++++++++++++++++++ libraries/liblmdb/mdb_stat.1 | 9 +- libraries/liblmdb/mdb_stat.c | 9 +- libraries/liblmdb/midl.c | 2 +- libraries/liblmdb/midl.h | 2 +- 11 files changed, 382 insertions(+), 16 deletions(-) create mode 100644 libraries/liblmdb/mdb_dump.1 create mode 100644 libraries/liblmdb/mdb_dump.c diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile index 9160c5731e..a34a9deb14 100644 --- a/libraries/liblmdb/Makefile +++ b/libraries/liblmdb/Makefile @@ -29,8 +29,8 @@ prefix = /usr/local IHDRS = lmdb.h ILIBS = liblmdb.a liblmdb.so -IPROGS = mdb_stat mdb_copy -IDOCS = mdb_stat.1 mdb_copy.1 +IPROGS = mdb_stat mdb_copy mdb_dump +IDOCS = mdb_stat.1 mdb_copy.1 mdb_dump.1 PROGS = $(IPROGS) mtest mtest2 mtest3 mtest4 mtest5 all: $(ILIBS) $(PROGS) @@ -56,6 +56,7 @@ liblmdb.so: mdb.o midl.o mdb_stat: mdb_stat.o liblmdb.a mdb_copy: mdb_copy.o liblmdb.a +mdb_dump: mdb_dump.o liblmdb.a mtest: mtest.o liblmdb.a mtest2: mtest2.o liblmdb.a mtest3: mtest3.o liblmdb.a diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h index 5b27973b60..c7a259049b 100644 --- a/libraries/liblmdb/lmdb.h +++ b/libraries/liblmdb/lmdb.h @@ -184,7 +184,7 @@ typedef int mdb_filehandle_t; /** Library minor version */ #define MDB_VERSION_MINOR 9 /** Library patch version */ -#define MDB_VERSION_PATCH 13 +#define MDB_VERSION_PATCH 14 /** Combine args a,b,c into a single integer for easy version comparisons */ #define MDB_VERINT(a,b,c) (((a) << 24) | ((b) << 16) | (c)) @@ -194,10 +194,10 @@ typedef int mdb_filehandle_t; MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH) /** The release date of this library version */ -#define MDB_VERSION_DATE "June 13, 2014" +#define MDB_VERSION_DATE "June 20, 2014" /** A stringifier for the version info */ -#define MDB_VERSTR(a,b,c,d) "MDB " #a "." #b "." #c ": (" d ")" +#define MDB_VERSTR(a,b,c,d) "LMDB " #a "." #b "." #c ": (" d ")" /** A helper for the stringifier macro */ #define MDB_VERFOO(a,b,c,d) MDB_VERSTR(a,b,c,d) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index fdc03864df..22629018c1 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -5,7 +5,7 @@ * BerkeleyDB API, but much simplified. */ /* - * Copyright 2011-2013 Howard Chu, Symas Corp. + * Copyright 2011-2014 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/libraries/liblmdb/mdb_copy.1 b/libraries/liblmdb/mdb_copy.1 index b4ab7531a8..58c6c5b60c 100644 --- a/libraries/liblmdb/mdb_copy.1 +++ b/libraries/liblmdb/mdb_copy.1 @@ -1,11 +1,13 @@ -.TH MDB_COPY 1 "2012/12/12" "LMDB 0.9.5" -.\" Copyright 2012 Howard Chu, Symas Corp. All Rights Reserved. +.TH MDB_COPY 1 "2014/06/20" "LMDB 0.9.14" +.\" Copyright 2012-2014 Howard Chu, Symas Corp. All Rights Reserved. .\" Copying restrictions apply. See COPYRIGHT/LICENSE. .SH NAME mdb_copy \- LMDB environment copy tool .SH SYNOPSIS .B mdb_copy [\c +.BR \-V ] +[\c .BR \-n ] .B srcpath [\c @@ -24,6 +26,10 @@ for storing the backup. Otherwise, the backup will be written to stdout. .SH OPTIONS +.TP +.BR \-V +Write the library version number to the standard output, and exit. +.TP .BR \-n Open LDMB environment(s) which do not use subdirectories. diff --git a/libraries/liblmdb/mdb_copy.c b/libraries/liblmdb/mdb_copy.c index bbf0dc902b..87525c0682 100644 --- a/libraries/liblmdb/mdb_copy.c +++ b/libraries/liblmdb/mdb_copy.c @@ -37,12 +37,15 @@ int main(int argc,char * argv[]) for (; argc > 1 && argv[1][0] == '-'; argc--, argv++) { if (argv[1][1] == 'n' && argv[1][2] == '\0') flags |= MDB_NOSUBDIR; - else + else if (argv[1][1] == 'V' && argv[1][2] == '\0') { + printf("%s\n", MDB_VERSION_STRING); + exit(0); + } else argc = 0; } if (argc<2 || argc>3) { - fprintf(stderr, "usage: %s [-n] srcpath [dstpath]\n", progname); + fprintf(stderr, "usage: %s [-V] [-n] srcpath [dstpath]\n", progname); exit(EXIT_FAILURE); } diff --git a/libraries/liblmdb/mdb_dump.1 b/libraries/liblmdb/mdb_dump.1 new file mode 100644 index 0000000000..a89dd5252d --- /dev/null +++ b/libraries/liblmdb/mdb_dump.1 @@ -0,0 +1,70 @@ +.TH MDB_DUMP 1 "2014/06/20" "LMDB 0.9.14" +.\" Copyright 2014 Howard Chu, Symas Corp. All Rights Reserved. +.\" Copying restrictions apply. See COPYRIGHT/LICENSE. +.SH NAME +mdb_dump \- LMDB environment export tool +.SH SYNOPSIS +.B mdb_dump +.BR \ envpath +[\c +.BR \-V ] +[\c +.BI \-f \ file\fR] +[\c +.BR \-l ] +[\c +.BR \-n ] +[\c +.BR \-p ] +[\c +.BR \-a \ | +.BI \-s \ subdb\fR] +.SH DESCRIPTION +The +.B mdb_dump +utility reads a database and writes its contents to the +standard output using a portable flat-text format +understood by the +.BR mdb_load (1) +utility. +.SH OPTIONS +.TP +.BR \-V +Write the library version number to the standard output, and exit. +.TP +.BR \-f \ file +Write to the specified file instead of to the standard output. +.TP +.BR \-l +List the databases stored in the environment. Just the +names will be listed, no data will be output. +.TP +.BR \-n +Dump an LMDB database which does not use subdirectories. +.TP +.BR \-p +If characters in either the key or data items are printing characters (as defined by isprint(3)), output them directly. This option permits users to use standard text editors and tools to modify the contents of databases. + +Note: different systems may have different notions about what characters are considered printing characters, and databases dumped in this manner may be less portable to external systems. +.TP +.BR \-a +Dump all of the subdatabases in the environment. +.TP +.BR \-s \ subdb +Dump a specific subdatabase. If no database is specified, only the main database is dumped. +.SH DIAGNOSTICS +Exit status is zero if no errors occur. +Errors result in a non-zero exit status and +a diagnostic message being written to standard error. + +Dumping and reloading databases that use user-defined comparison functions will result in new databases that use the +default comparison functions. \fB In this case it is quite likely that the reloaded database will be damaged beyond +repair permitting neither record storage nor retrieval.\fP + +The only available workaround is to modify the source for the +.BR mdb_load (1) +utility to load the database using the correct comparison functions. +.SH "SEE ALSO" +.BR mdb_load (1) +.SH AUTHOR +Howard Chu of Symas Corporation diff --git a/libraries/liblmdb/mdb_dump.c b/libraries/liblmdb/mdb_dump.c new file mode 100644 index 0000000000..5242519ebc --- /dev/null +++ b/libraries/liblmdb/mdb_dump.c @@ -0,0 +1,276 @@ +/* mdb_dump.c - memory-mapped database dump tool */ +/* + * Copyright 2011-2014 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ +#include +#include +#include +#include +#include +#include +#include "lmdb.h" + +#define PRINT 1 +static int mode; + +typedef struct flagbit { + int bit; + char *name; +} flagbit; + +flagbit dbflags[] = { + { MDB_REVERSEKEY, "reversekey" }, + { MDB_DUPSORT, "dupsort" }, + { MDB_INTEGERKEY, "integerkey" }, + { MDB_DUPFIXED, "dupfixed" }, + { MDB_INTEGERDUP, "integerdup" }, + { MDB_REVERSEDUP, "reversedup" }, + { 0, NULL } +}; + +static const char hexc[] = "0123456789abcdef"; + +static void hex(unsigned char c) +{ + putchar(hexc[c >> 4]); + putchar(hexc[c & 0xf]); +} + +static void text(MDB_val *v) +{ + unsigned char *c, *end; + + putchar(' '); + c = v->mv_data; + end = c + v->mv_size; + while (c < end) { + if (isprint(*c)) { + putchar(*c); + } else { + putchar('\\'); + hex(*c); + } + c++; + } + putchar('\n'); +} + +static void byte(MDB_val *v) +{ + unsigned char *c, *end; + + putchar(' '); + c = v->mv_data; + end = c + v->mv_size; + while (c < end) { + hex(*c++); + } + putchar('\n'); +} + +/* Dump in BDB-compatible format */ +static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name) +{ + MDB_cursor *mc; + MDB_stat ms; + MDB_val key, data; + unsigned int flags; + int rc, i; + + rc = mdb_dbi_flags(txn, dbi, &flags); + if (rc) return rc; + + rc = mdb_stat(txn, dbi, &ms); + if (rc) return rc; + + printf("VERSION=3\n"); + printf("format=%s\n", mode & PRINT ? "print" : "bytevalue"); + if (name) + printf("database=%s\n", name); + printf("type=btree\n"); + + if (flags & MDB_DUPSORT) + printf("duplicates=1\n"); + + for (i=0; dbflags[i].bit; i++) + if (flags & dbflags[i].bit) + printf("%s=1\n", dbflags[i].name); + + printf("db_pagesize=%d\n", ms.ms_psize); + printf("HEADER=END\n"); + + rc = mdb_cursor_open(txn, dbi, &mc); + if (rc) return rc; + + while ((rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT) == MDB_SUCCESS)) { + if (mode & PRINT) { + text(&key); + text(&data); + } else { + byte(&key); + byte(&data); + } + } + printf("DATA=END\n"); + if (rc == MDB_NOTFOUND) + rc = MDB_SUCCESS; + + return rc; +} + +static void usage(char *prog) +{ + fprintf(stderr, "usage: %s dbpath [-V] [-f output] [-l] [-n] [-p] [-a|-s subdb]\n", prog); + exit(EXIT_FAILURE); +} + +int main(int argc, char *argv[]) +{ + int i, rc; + MDB_env *env; + MDB_txn *txn; + MDB_dbi dbi; + char *prog = argv[0]; + char *envname; + char *subname = NULL; + int alldbs = 0, envflags = 0, list = 0; + + if (argc < 2) { + usage(prog); + } + + /* -a: dump main DB and all subDBs + * -s: dump only the named subDB + * -n: use NOSUBDIR flag on env_open + * -p: use printable characters + * -f: write to file instead of stdout + * -V: print version and exit + * (default) dump only the main DB + */ + while ((i = getopt(argc, argv, "af:lnps:V")) != EOF) { + switch(i) { + case 'V': + printf("%s\n", MDB_VERSION_STRING); + exit(0); + break; + case 'l': + list = 1; + /*FALLTHROUGH*/; + case 'a': + if (subname) + usage(prog); + alldbs++; + break; + case 'f': + if (freopen(optarg, "w", stdout) == NULL) { + fprintf(stderr, "%s: %s: reopen: %s\n", + prog, optarg, strerror(errno)); + exit(EXIT_FAILURE); + } + break; + case 'n': + envflags |= MDB_NOSUBDIR; + break; + case 'p': + mode |= PRINT; + break; + case 's': + if (alldbs) + usage(prog); + subname = optarg; + break; + default: + usage(prog); + } + } + + if (optind != argc - 1) + usage(prog); + + envname = argv[optind]; + rc = mdb_env_create(&env); + + if (alldbs || subname) { + mdb_env_set_maxdbs(env, 2); + } + + rc = mdb_env_open(env, envname, envflags | MDB_RDONLY, 0664); + if (rc) { + printf("mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; + } + + rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (rc) { + printf("mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; + } + + rc = mdb_open(txn, subname, 0, &dbi); + if (rc) { + printf("mdb_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + + if (alldbs) { + MDB_cursor *cursor; + MDB_val key; + int count = 0; + + rc = mdb_cursor_open(txn, dbi, &cursor); + if (rc) { + printf("mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + while ((rc = mdb_cursor_get(cursor, &key, NULL, MDB_NEXT_NODUP)) == 0) { + char *str; + MDB_dbi db2; + if (memchr(key.mv_data, '\0', key.mv_size)) + continue; + count++; + str = malloc(key.mv_size+1); + memcpy(str, key.mv_data, key.mv_size); + str[key.mv_size] = '\0'; + rc = mdb_open(txn, str, 0, &db2); + if (rc == MDB_SUCCESS) { + if (list) { + printf("%s\n", str); + list++; + } else { + rc = dumpit(txn, db2, str); + } + mdb_close(env, db2); + } + free(str); + if (rc) continue; + } + mdb_cursor_close(cursor); + if (!count) { + fprintf(stderr, "%s: %s does not contain multiple databases\n", prog, envname); + rc = MDB_NOTFOUND; + } else if (rc == MDB_NOTFOUND) { + rc = MDB_SUCCESS; + } + } else { + rc = dumpit(txn, dbi, subname); + } + if (rc && rc != MDB_NOTFOUND) + fprintf(stderr, "%s: %s: %s\n", prog, envname, mdb_strerror(rc)); + + mdb_close(env, dbi); +txn_abort: + mdb_txn_abort(txn); +env_close: + mdb_env_close(env); + + return rc ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/libraries/liblmdb/mdb_stat.1 b/libraries/liblmdb/mdb_stat.1 index 3622772ddf..3d8d461d99 100644 --- a/libraries/liblmdb/mdb_stat.1 +++ b/libraries/liblmdb/mdb_stat.1 @@ -1,5 +1,5 @@ -.TH MDB_STAT 1 "2012/12/12" "LMDB 0.9.5" -.\" Copyright 2012 Howard Chu, Symas Corp. All Rights Reserved. +.TH MDB_STAT 1 "2014/06/20" "LMDB 0.9.14" +.\" Copyright 2012-2014 Howard Chu, Symas Corp. All Rights Reserved. .\" Copying restrictions apply. See COPYRIGHT/LICENSE. .SH NAME mdb_stat \- LMDB environment status tool @@ -7,6 +7,8 @@ mdb_stat \- LMDB environment status tool .B mdb_stat .BR \ envpath [\c +.BR \-V ] +[\c .BR \-e ] [\c .BR \-f [ f [ f ]]] @@ -23,6 +25,9 @@ The utility displays the status of an LMDB environment. .SH OPTIONS .TP +.BR \-V +Write the library version number to the standard output, and exit. +.TP .BR \-e Display information about the database environment. .TP diff --git a/libraries/liblmdb/mdb_stat.c b/libraries/liblmdb/mdb_stat.c index 40bd4ccf1d..eac2c60274 100644 --- a/libraries/liblmdb/mdb_stat.c +++ b/libraries/liblmdb/mdb_stat.c @@ -37,7 +37,7 @@ static void prstat(MDB_stat *ms) static void usage(char *prog) { - fprintf(stderr, "usage: %s dbpath [-n] [-e] [-r[r]] [-f[f[f]]] [-a|-s subdb]\n", prog); + fprintf(stderr, "usage: %s dbpath [-V] [-n] [-e] [-r[r]] [-f[f[f]]] [-a|-s subdb]\n", prog); exit(EXIT_FAILURE); } @@ -64,10 +64,15 @@ int main(int argc, char *argv[]) * -f: print freelist info * -r: print reader info * -n: use NOSUBDIR flag on env_open + * -V: print version and exit * (default) print stat of only the main DB */ - while ((i = getopt(argc, argv, "aefnrs:")) != EOF) { + while ((i = getopt(argc, argv, "Vaefnrs:")) != EOF) { switch(i) { + case 'V': + printf("%s\n", MDB_VERSION_STRING); + exit(0); + break; case 'a': if (subname) usage(prog); diff --git a/libraries/liblmdb/midl.c b/libraries/liblmdb/midl.c index efcaad4750..b39d463c56 100644 --- a/libraries/liblmdb/midl.c +++ b/libraries/liblmdb/midl.c @@ -3,7 +3,7 @@ /* $OpenLDAP$ */ /* This work is part of OpenLDAP Software . * - * Copyright 2000-2013 The OpenLDAP Foundation. + * Copyright 2000-2014 The OpenLDAP Foundation. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/libraries/liblmdb/midl.h b/libraries/liblmdb/midl.h index 635cd29e0e..f2bb4338de 100644 --- a/libraries/liblmdb/midl.h +++ b/libraries/liblmdb/midl.h @@ -11,7 +11,7 @@ /* $OpenLDAP$ */ /* This work is part of OpenLDAP Software . * - * Copyright 2000-2013 The OpenLDAP Foundation. + * Copyright 2000-2014 The OpenLDAP Foundation. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From 21b51cb746aa455e3c6398f35b56b9117e657d64 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Fri, 20 Jun 2014 08:49:59 -0700 Subject: [PATCH 02/16] Add mdb_load --- libraries/liblmdb/Makefile | 5 +- libraries/liblmdb/mdb_dump.1 | 15 +- libraries/liblmdb/mdb_load.1 | 77 +++++++ libraries/liblmdb/mdb_load.c | 397 +++++++++++++++++++++++++++++++++++ 4 files changed, 487 insertions(+), 7 deletions(-) create mode 100644 libraries/liblmdb/mdb_load.1 create mode 100644 libraries/liblmdb/mdb_load.c diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile index a34a9deb14..b65c9b9a8b 100644 --- a/libraries/liblmdb/Makefile +++ b/libraries/liblmdb/Makefile @@ -29,8 +29,8 @@ prefix = /usr/local IHDRS = lmdb.h ILIBS = liblmdb.a liblmdb.so -IPROGS = mdb_stat mdb_copy mdb_dump -IDOCS = mdb_stat.1 mdb_copy.1 mdb_dump.1 +IPROGS = mdb_stat mdb_copy mdb_dump mdb_load +IDOCS = mdb_stat.1 mdb_copy.1 mdb_dump.1 mdb_load.1 PROGS = $(IPROGS) mtest mtest2 mtest3 mtest4 mtest5 all: $(ILIBS) $(PROGS) @@ -57,6 +57,7 @@ liblmdb.so: mdb.o midl.o mdb_stat: mdb_stat.o liblmdb.a mdb_copy: mdb_copy.o liblmdb.a mdb_dump: mdb_dump.o liblmdb.a +mdb_load: mdb_load.o liblmdb.a mtest: mtest.o liblmdb.a mtest2: mtest2.o liblmdb.a mtest3: mtest3.o liblmdb.a diff --git a/libraries/liblmdb/mdb_dump.1 b/libraries/liblmdb/mdb_dump.1 index a89dd5252d..6fcc930811 100644 --- a/libraries/liblmdb/mdb_dump.1 +++ b/libraries/liblmdb/mdb_dump.1 @@ -43,9 +43,13 @@ names will be listed, no data will be output. Dump an LMDB database which does not use subdirectories. .TP .BR \-p -If characters in either the key or data items are printing characters (as defined by isprint(3)), output them directly. This option permits users to use standard text editors and tools to modify the contents of databases. +If characters in either the key or data items are printing characters (as +defined by isprint(3)), output them directly. This option permits users to +use standard text editors and tools to modify the contents of databases. -Note: different systems may have different notions about what characters are considered printing characters, and databases dumped in this manner may be less portable to external systems. +Note: different systems may have different notions about what characters +are considered printing characters, and databases dumped in this manner may +be less portable to external systems. .TP .BR \-a Dump all of the subdatabases in the environment. @@ -57,9 +61,10 @@ Exit status is zero if no errors occur. Errors result in a non-zero exit status and a diagnostic message being written to standard error. -Dumping and reloading databases that use user-defined comparison functions will result in new databases that use the -default comparison functions. \fB In this case it is quite likely that the reloaded database will be damaged beyond -repair permitting neither record storage nor retrieval.\fP +Dumping and reloading databases that use user-defined comparison functions +will result in new databases that use the default comparison functions. +\fBIn this case it is quite likely that the reloaded database will be +damaged beyond repair permitting neither record storage nor retrieval.\fP The only available workaround is to modify the source for the .BR mdb_load (1) diff --git a/libraries/liblmdb/mdb_load.1 b/libraries/liblmdb/mdb_load.1 new file mode 100644 index 0000000000..7280240103 --- /dev/null +++ b/libraries/liblmdb/mdb_load.1 @@ -0,0 +1,77 @@ +.TH MDB_LOAD 1 "2014/06/20" "LMDB 0.9.14" +.\" Copyright 2014 Howard Chu, Symas Corp. All Rights Reserved. +.\" Copying restrictions apply. See COPYRIGHT/LICENSE. +.SH NAME +mdb_load \- LMDB environment import tool +.SH SYNOPSIS +.B mdb_load +.BR \ envpath +[\c +.BR \-V ] +[\c +.BI \-f \ file\fR] +[\c +.BR \-n ] +[\c +.BI \-s \ subdb\fR] +[\c +.BR \-N ] +[\c +.BR \-T ] +.SH DESCRIPTION +The +.B mdb_load +utility reads from the standard input and loads it into the +LMDB environment +.BR envpath . + +The input to +.B mdb_load +must be in the output format specified by the +.BR mdb_dump (1) +utility or as specified by the +.B -T +option below. +.SH OPTIONS +.TP +.BR \-V +Write the library version number to the standard output, and exit. +.TP +.BR \-f \ file +Read from the specified file instead of from the standard input. +.TP +.BR \-n +Load an LMDB database which does not use subdirectories. +.TP +.BR \-s \ subdb +Load a specific subdatabase. If no database is specified, data is loaded into the main database. +.TP +.BR \-N +Don't overwrite existing records when loading into an already existing database; just skip them. +.TP +.BR \-T +Load data from simple text files. The input must be paired lines of text, where the first +line of the pair is the key item, and the second line of the pair is its corresponding +data item. + +A simple escape mechanism, where newline and backslash (\\) characters are special, is +applied to the text input. Newline characters are interpreted as record separators. +Backslash characters in the text will be interpreted in one of two ways: If the backslash +character precedes another backslash character, the pair will be interpreted as a literal +backslash. If the backslash character precedes any other character, the two characters +following the backslash will be interpreted as a hexadecimal specification of a single +character; for example, \\0a is a newline character in the ASCII character set. + +For this reason, any backslash or newline characters that naturally occur in the text +input must be escaped to avoid misinterpretation by +.BR mdb_load . + +.SH DIAGNOSTICS +Exit status is zero if no errors occur. +Errors result in a non-zero exit status and +a diagnostic message being written to standard error. + +.SH "SEE ALSO" +.BR mdb_dump (1) +.SH AUTHOR +Howard Chu of Symas Corporation diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c new file mode 100644 index 0000000000..27ffc2801f --- /dev/null +++ b/libraries/liblmdb/mdb_load.c @@ -0,0 +1,397 @@ +/* mdb_load.c - memory-mapped database load tool */ +/* + * Copyright 2011-2014 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ +#include +#include +#include +#include +#include +#include +#include "lmdb.h" + +#define PRINT 1 +#define NOHDR 2 +static int mode; + +static char *subname = NULL; + +static size_t lineno; +static int version; + +static int flags; + +static char *prog; + +static int eof; + +static MDB_val kbuf, dbuf; + +#define STRLENOF(s) (sizeof(s)-1) + +typedef struct flagbit { + int bit; + char *name; + int len; +} flagbit; + +#define S(s) s, STRLENOF(s) + +flagbit dbflags[] = { + { MDB_REVERSEKEY, S("reversekey") }, + { MDB_DUPSORT, S("dupsort") }, + { MDB_INTEGERKEY, S("integerkey") }, + { MDB_DUPFIXED, S("dupfixed") }, + { MDB_INTEGERDUP, S("integerdup") }, + { MDB_REVERSEDUP, S("reversedup") }, + { 0, NULL, 0 } +}; + +static const char hexc[] = "0123456789abcdef"; + +static void readhdr() +{ + char *ptr; + + while (fgets(dbuf.mv_data, dbuf.mv_size, stdin) != NULL) { + lineno++; + if (!strncmp(dbuf.mv_data, "VERSION=", STRLENOF("VERSION="))) { + version=atoi(dbuf.mv_data+STRLENOF("VERSION=")); + if (version > 3) { + fprintf(stderr, "%s: line %zd: unsupported VERSION %d\n", + prog, lineno, version); + exit(EXIT_FAILURE); + } + } else if (!strncmp(dbuf.mv_data, "HEADER=END", STRLENOF("HEADER=END"))) { + break; + } else if (!strncmp(dbuf.mv_data, "format=", STRLENOF("format="))) { + if (!strncmp(dbuf.mv_data+STRLENOF("FORMAT="), "print", STRLENOF("print"))) + mode |= PRINT; + else if (strncmp(dbuf.mv_data+STRLENOF("FORMAT="), "bytevalue", STRLENOF("bytevalue"))) { + fprintf(stderr, "%s: line %zd: unsupported FORMAT %s\n", + prog, lineno, (char *)dbuf.mv_data+STRLENOF("FORMAT=")); + exit(EXIT_FAILURE); + } + } else if (!strncmp(dbuf.mv_data, "database=", STRLENOF("database="))) { + ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size); + if (ptr) *ptr = '\0'; + if (subname) free(subname); + subname = strdup(dbuf.mv_data+STRLENOF("database=")); + } else if (!strncmp(dbuf.mv_data, "type=", STRLENOF("type="))) { + if (strncmp(dbuf.mv_data+STRLENOF("type="), "btree", STRLENOF("btree"))) { + fprintf(stderr, "%s: line %zd: unsupported type %s\n", + prog, lineno, (char *)dbuf.mv_data+STRLENOF("type=")); + exit(EXIT_FAILURE); + } + } else { + int i; + for (i=0; dbflags[i].bit; i++) { + if (!strncmp(dbuf.mv_data, dbflags[i].name, dbflags[i].len) && + ((char *)dbuf.mv_data)[dbflags[i].len] == '=') { + flags |= dbflags[i].bit; + break; + } + } + if (!dbflags[i].bit) { + ptr = memchr(dbuf.mv_data, '=', dbuf.mv_size); + if (!ptr) { + fprintf(stderr, "%s: line %zd: unexpected format\n", + prog, lineno); + exit(EXIT_FAILURE); + } else { + *ptr = '\0'; + fprintf(stderr, "%s: line %zd: unrecognized keyword ignored: %s\n", + prog, lineno, (char *)dbuf.mv_data); + } + } + } + } +} + +static void badend() +{ + fprintf(stderr, "%s: line %zd: unexpected end of input\n", + prog, lineno); +} + +static int unhex(unsigned char *c2) +{ + int x, c; + x = *c2++ & 0x4f; + if (x & 0x40) + x -= 54; + c = x << 4; + x = *c2 & 0x4f; + if (x & 0x40) + x -= 54; + c |= x; + return c; +} + +static int readline(MDB_val *out, MDB_val *buf) +{ + unsigned char *c1, *c2, *end; + size_t len; + int c; + + if (!(mode & NOHDR)) { + c = fgetc(stdin); + if (c == EOF) { + eof = 1; + return EOF; + } + if (c != ' ') { + if (fgets(buf->mv_data, buf->mv_size, stdin) == NULL) { +badend: + eof = 1; + badend(); + return EOF; + } + if (c == 'D' && !strncmp(buf->mv_data, "ATA=END", STRLENOF("ATA=END"))) + return EOF; + goto badend; + } + } + if (fgets(buf->mv_data, buf->mv_size, stdin) == NULL) { + eof = 1; + return EOF; + } + lineno++; + + c1 = buf->mv_data; + len = strlen((char *)c1); + + /* Is buffer too short? */ + while (c1[len-1] != '\n') { + buf->mv_data = realloc(buf->mv_data, buf->mv_size*2); + if (!buf->mv_data) { + eof = 1; + fprintf(stderr, "%s: line %zd: out of memory, line too long\n", + prog, lineno); + return EOF; + } + c1 = buf->mv_data; + c1 += buf->mv_size; + if (fgets((char *)c1, buf->mv_size, stdin) == NULL) { + eof = 1; + badend(); + return EOF; + } + buf->mv_size *= 2; + len = strlen((char *)c1); + } + c1 = c2 = buf->mv_data; + len = strlen((char *)c1); + c1[--len] = '\0'; + end = c1 + len; + + if (mode & PRINT) { + while (c2 < end) { + if (*c2 == '\\') { + if (c2[1] == '\\') { + c1++; c2 += 2; + } else { + if (c2+3 >= end || !isxdigit(c2[1]) || !isxdigit(c2[2])) { + eof = 1; + badend(); + return EOF; + } + *c1++ = unhex(++c2); + c2 += 2; + } + } else { + c1++; c2++; + } + } + } else { + /* odd length not allowed */ + if (len & 1) { + eof = 1; + badend(); + return EOF; + } + while (c2 < end) { + if (!isxdigit(*c2) || !isxdigit(c2[1])) { + eof = 1; + badend(); + return EOF; + } + *c1++ = unhex(c2); + c2 += 2; + } + } + c2 = out->mv_data = buf->mv_data; + out->mv_size = c1 - c2; + + return 0; +} + +static void usage() +{ + fprintf(stderr, "usage: %s dbpath [-V] [-f input] [-n] [-s name] [-N] [-T]\n", prog); + exit(EXIT_FAILURE); +} + +int main(int argc, char *argv[]) +{ + int i, rc; + MDB_env *env; + MDB_txn *txn; + MDB_cursor *mc; + MDB_dbi dbi; + char *envname; + int envflags = 0, putflags = 0; + + prog = argv[0]; + + if (argc < 2) { + usage(prog); + } + + /* -f: load file instead of stdin + * -n: use NOSUBDIR flag on env_open + * -s: load into named subDB + * -N: use NOOVERWRITE on puts + * -T: read plaintext + * -V: print version and exit + */ + while ((i = getopt(argc, argv, "f:ns:NTV")) != EOF) { + switch(i) { + case 'V': + printf("%s\n", MDB_VERSION_STRING); + exit(0); + break; + case 'f': + if (freopen(optarg, "r", stdin) == NULL) { + fprintf(stderr, "%s: %s: reopen: %s\n", + prog, optarg, strerror(errno)); + exit(EXIT_FAILURE); + } + break; + case 'n': + envflags |= MDB_NOSUBDIR; + break; + case 's': + subname = strdup(optarg); + break; + case 'N': + putflags = MDB_NOOVERWRITE|MDB_NODUPDATA; + break; + case 'T': + mode |= NOHDR; + break; + default: + usage(prog); + } + } + + if (optind != argc - 1) + usage(prog); + + envname = argv[optind]; + rc = mdb_env_create(&env); + + if (subname) { + mdb_env_set_maxdbs(env, 2); + } + + rc = mdb_env_open(env, envname, envflags, 0664); + if (rc) { + printf("mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; + } + + kbuf.mv_size = mdb_env_get_maxkeysize(env) * 2 + 2; + kbuf.mv_data = malloc(kbuf.mv_size); + dbuf.mv_size = 4096; + dbuf.mv_data = malloc(dbuf.mv_size); + + while(!eof) { + MDB_val key, data; + int batch = 0; + flags = 0; + + if (!(mode & NOHDR)) + readhdr(); + + rc = mdb_txn_begin(env, NULL, 0, &txn); + if (rc) { + printf("mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; + } + + rc = mdb_open(txn, subname, flags, &dbi); + if (rc) { + printf("mdb_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + + rc = mdb_cursor_open(txn, dbi, &mc); + if (rc) { + printf("mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + + while(1) { + rc = readline(&key, &kbuf); + if (rc == EOF) + break; + if (rc) + goto txn_abort; + + rc = readline(&data, &dbuf); + if (rc) + goto txn_abort; + + rc = mdb_cursor_put(mc, &key, &data, putflags); + if (rc == MDB_KEYEXIST && putflags) + continue; + if (rc) + goto txn_abort; + batch++; + if (batch == 100) { + rc = mdb_txn_commit(txn); + if (rc) { + fprintf(stderr, "%s: line %zd: txn_commit: %s\n", + prog, lineno, mdb_strerror(rc)); + goto env_close; + } + rc = mdb_txn_begin(env, NULL, 0, &txn); + if (rc) { + printf("mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; + } + rc = mdb_cursor_open(txn, dbi, &mc); + if (rc) { + printf("mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + batch = 0; + } + } + rc = mdb_txn_commit(txn); + txn = NULL; + if (rc) { + fprintf(stderr, "%s: line %zd: txn_commit: %s\n", + prog, lineno, mdb_strerror(rc)); + goto env_close; + } + mdb_dbi_close(env, dbi); + } + +txn_abort: + mdb_txn_abort(txn); +env_close: + mdb_env_close(env); + + return rc ? EXIT_FAILURE : EXIT_SUCCESS; +} From 1a72b19e26d226eaec26a7f9cd231337f20bf4da Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Fri, 20 Jun 2014 08:56:52 -0700 Subject: [PATCH 03/16] Fix unhex --- libraries/liblmdb/mdb_load.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c index 27ffc2801f..967b0ff249 100644 --- a/libraries/liblmdb/mdb_load.c +++ b/libraries/liblmdb/mdb_load.c @@ -128,11 +128,11 @@ static int unhex(unsigned char *c2) int x, c; x = *c2++ & 0x4f; if (x & 0x40) - x -= 54; + x -= 55; c = x << 4; x = *c2 & 0x4f; if (x & 0x40) - x -= 54; + x -= 55; c |= x; return c; } From c05f45b7dd83ab582c99a720fd44d6a6ba8ddf32 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Fri, 20 Jun 2014 09:00:33 -0700 Subject: [PATCH 04/16] Fix multiple subDBs --- libraries/liblmdb/mdb_load.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c index 967b0ff249..326e8e78a7 100644 --- a/libraries/liblmdb/mdb_load.c +++ b/libraries/liblmdb/mdb_load.c @@ -300,9 +300,7 @@ int main(int argc, char *argv[]) envname = argv[optind]; rc = mdb_env_create(&env); - if (subname) { - mdb_env_set_maxdbs(env, 2); - } + mdb_env_set_maxdbs(env, 2); rc = mdb_env_open(env, envname, envflags, 0664); if (rc) { @@ -329,7 +327,7 @@ int main(int argc, char *argv[]) goto env_close; } - rc = mdb_open(txn, subname, flags, &dbi); + rc = mdb_open(txn, subname, flags|MDB_CREATE, &dbi); if (rc) { printf("mdb_open failed, error %d %s\n", rc, mdb_strerror(rc)); goto txn_abort; From 0d013609fe754dd065faf77297d8d5b1c93bb672 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Fri, 20 Jun 2014 09:03:41 -0700 Subject: [PATCH 05/16] Fix lineno count --- libraries/liblmdb/mdb_load.c | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c index 326e8e78a7..8ac21d41fd 100644 --- a/libraries/liblmdb/mdb_load.c +++ b/libraries/liblmdb/mdb_load.c @@ -150,6 +150,7 @@ static int readline(MDB_val *out, MDB_val *buf) return EOF; } if (c != ' ') { + lineno++; if (fgets(buf->mv_data, buf->mv_size, stdin) == NULL) { badend: eof = 1; From 79e8462d748543f7dc36a3ff3fa8263c62b34c3f Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Fri, 20 Jun 2014 17:56:04 -0700 Subject: [PATCH 06/16] Fix mdb_node_move Was leaving stale info in dst cursor --- libraries/liblmdb/mdb.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 22629018c1..551519c150 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -7054,20 +7054,20 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst) MDB_node *s2; MDB_val bkey; /* must find the lowest key below dst */ - rc = mdb_page_search_lowest(cdst); + mdb_cursor_copy(cdst, &mn); + rc = mdb_page_search_lowest(&mn); if (rc) return rc; - if (IS_LEAF2(cdst->mc_pg[cdst->mc_top])) { - bkey.mv_size = cdst->mc_db->md_pad; - bkey.mv_data = LEAF2KEY(cdst->mc_pg[cdst->mc_top], 0, bkey.mv_size); + if (IS_LEAF2(mn.mc_pg[mn.mc_top])) { + bkey.mv_size = mn.mc_db->md_pad; + bkey.mv_data = LEAF2KEY(mn.mc_pg[mn.mc_top], 0, bkey.mv_size); } else { - s2 = NODEPTR(cdst->mc_pg[cdst->mc_top], 0); + s2 = NODEPTR(mn.mc_pg[mn.mc_top], 0); bkey.mv_size = NODEKSZ(s2); bkey.mv_data = NODEKEY(s2); } - cdst->mc_snum = snum--; - cdst->mc_top = snum; - mdb_cursor_copy(cdst, &mn); + mn.mc_snum = snum--; + mn.mc_top = snum; mn.mc_ki[snum] = 0; rc = mdb_update_key(&mn, &bkey); if (rc) From eebbd22c2ff9d6088e307c960be56baf35e1ab0d Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Fri, 20 Jun 2014 23:40:44 -0700 Subject: [PATCH 07/16] Fix mdb_page_merge Similar to prev commit for mdb_node_move, the call of mdb_page_serch_lowest() was leaving csrc unusable. --- libraries/liblmdb/mdb.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 551519c150..99e68727e0 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -7215,22 +7215,21 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst) for (i = 0; i < NUMKEYS(csrc->mc_pg[csrc->mc_top]); i++, j++) { srcnode = NODEPTR(csrc->mc_pg[csrc->mc_top], i); if (i == 0 && IS_BRANCH(csrc->mc_pg[csrc->mc_top])) { - unsigned int snum = csrc->mc_snum; + MDB_cursor mn; MDB_node *s2; + mdb_cursor_copy(csrc, &mn); /* must find the lowest key below src */ - rc = mdb_page_search_lowest(csrc); + rc = mdb_page_search_lowest(&mn); if (rc) return rc; - if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { - key.mv_size = csrc->mc_db->md_pad; - key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], 0, key.mv_size); + if (IS_LEAF2(mn.mc_pg[mn.mc_top])) { + key.mv_size = mn.mc_db->md_pad; + key.mv_data = LEAF2KEY(mn.mc_pg[mn.mc_top], 0, key.mv_size); } else { - s2 = NODEPTR(csrc->mc_pg[csrc->mc_top], 0); + s2 = NODEPTR(mn.mc_pg[mn.mc_top], 0); key.mv_size = NODEKSZ(s2); key.mv_data = NODEKEY(s2); } - csrc->mc_snum = snum--; - csrc->mc_top = snum; } else { key.mv_size = srcnode->mn_ksize; key.mv_data = NODEKEY(srcnode); From 7cab7b95e240446dd11c7e0951af8b2c3409765b Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Sat, 21 Jun 2014 03:30:34 -0700 Subject: [PATCH 08/16] Handle loose pages Pages that were dirtied and deleted in the same txn should be reused, instead of consuming freeDB pages. --- libraries/liblmdb/mdb.c | 75 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 70 insertions(+), 5 deletions(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 99e68727e0..c6985b009b 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -650,6 +650,7 @@ typedef struct MDB_page { #define P_DIRTY 0x10 /**< dirty page, also set for #P_SUBP pages */ #define P_LEAF2 0x20 /**< for #MDB_DUPFIXED records */ #define P_SUBP 0x40 /**< for #MDB_DUPSORT sub-pages */ +#define P_LOOSE 0x4000 /**< page was dirtied then freed, can be reused */ #define P_KEEP 0x8000 /**< leave this page alone during spill */ /** @} */ uint16_t mp_flags; /**< @ref mdb_page */ @@ -1021,6 +1022,7 @@ typedef struct MDB_xcursor { typedef struct MDB_pgstate { pgno_t *mf_pghead; /**< Reclaimed freeDB pages, or NULL before use */ txnid_t mf_pglast; /**< ID of last used record, or 0 if !mf_pghead */ + MDB_page *mf_pgloose; /**< Dirty pages that can be reused */ } MDB_pgstate; /** The database environment. */ @@ -1057,6 +1059,7 @@ struct MDB_env { MDB_pgstate me_pgstate; /**< state of old pages from freeDB */ # define me_pglast me_pgstate.mf_pglast # define me_pghead me_pgstate.mf_pghead +# define me_pgloose me_pgstate.mf_pgloose MDB_page *me_dpages; /**< list of malloc'd blocks for re-use */ /** IDL of pages that became unused in a write txn */ MDB_IDL me_free_pgs; @@ -1485,7 +1488,6 @@ mdb_page_malloc(MDB_txn *txn, unsigned num) } return ret; } - /** Free a single page. * Saves single pages to a list, for future reuse. * (This is not used for multi-page overflow pages.) @@ -1525,6 +1527,23 @@ mdb_dlist_free(MDB_txn *txn) dl[0].mid = 0; } +/** Loosen a single page. + * Saves single pages to a list for future reuse + * in this same txn. It has been pulled from the freeDB + * and already resides on the dirty list, but has been + * deleted. Use these pages first before pulling again + * from the freeDB. + */ +static void +mdb_page_loose(MDB_env *env, MDB_page *mp) +{ + pgno_t *pp = (pgno_t *)mp->mp_ptrs; + *pp = mp->mp_pgno; + mp->mp_next = env->me_pgloose; + env->me_pgloose = mp; + mp->mp_flags |= P_LOOSE; +} + /** Set or clear P_KEEP in dirty, non-overflow, non-sub pages watched by txn. * @param[in] mc A cursor handle for the current operation. * @param[in] pflags Flags of the pages to update: @@ -1573,6 +1592,12 @@ mdb_pages_xkeep(MDB_cursor *mc, unsigned pflags, int all) break; } + /* Loose pages shouldn't be spilled */ + for (dp = txn->mt_env->me_pgloose; dp; dp=dp->mp_next) { + if ((dp->mp_flags & Mask) == pflags) + dp->mp_flags ^= P_KEEP; + } + if (all) { /* Mark dirty root pages */ for (i=0; imt_numdbs; i++) { @@ -1800,6 +1825,17 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) MDB_cursor_op op; MDB_cursor m2; + /* If there are any loose pages, just use them */ + if (num == 1 && env->me_pgloose) { + pgno_t *pp; + np = env->me_pgloose; + env->me_pgloose = np->mp_next; + pp = (pgno_t *)np->mp_ptrs; + np->mp_pgno = *pp; + *mp = np; + return MDB_SUCCESS; + } + *mp = NULL; /* If our dirty list is already full, we can't do anything */ @@ -2661,6 +2697,38 @@ mdb_freelist_save(MDB_txn *txn) return rc; } + /* Dispose of loose pages. Usually they will have all + * been used up by the time we get here. + */ + if (env->me_pgloose) { + MDB_page *mp = env->me_pgloose; + pgno_t *pp; + /* Just return them to freeDB */ + if (env->me_pghead) { + int i, j; + mop = env->me_pghead; + while(mp) { + pgno_t pg; + pp = (pgno_t *)mp->mp_ptrs; + pg = *pp; + j = mop[0] + 1; + for (i = mop[0]; i && mop[i] < pg; i--) + mop[j--] = mop[i]; + mop[j] = pg; + mop[0] += 1; + mp = mp->mp_next; + } + } else { + /* Oh well, they were wasted. Put on freelist */ + while(mp) { + pp = (pgno_t *)mp->mp_ptrs; + mdb_midl_append(&txn->mt_free_pgs, *pp); + mp = mp->mp_next; + } + } + env->me_pgloose = NULL; + } + /* MDB_RESERVE cancels meminit in ovpage malloc (when no WRITEMAP) */ clean_limit = (env->me_flags & (MDB_NOMEMINIT|MDB_WRITEMAP)) ? SSIZE_MAX : maxfree_1pg; @@ -7261,10 +7329,7 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst) } csrc->mc_top++; - rc = mdb_midl_append(&csrc->mc_txn->mt_free_pgs, - csrc->mc_pg[csrc->mc_top]->mp_pgno); - if (rc) - return rc; + mdb_page_loose(csrc->mc_txn->mt_env, csrc->mc_pg[csrc->mc_top]); if (IS_LEAF(csrc->mc_pg[csrc->mc_top])) csrc->mc_db->md_leaf_pages--; else From cf331ccb3dc482769d6ac8d75cc66249b0086d29 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Sat, 21 Jun 2014 08:34:45 -0700 Subject: [PATCH 09/16] Silence ptr arithmetic warnings --- libraries/liblmdb/mdb_load.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c index 8ac21d41fd..0cf02ada5e 100644 --- a/libraries/liblmdb/mdb_load.c +++ b/libraries/liblmdb/mdb_load.c @@ -65,7 +65,7 @@ static void readhdr() while (fgets(dbuf.mv_data, dbuf.mv_size, stdin) != NULL) { lineno++; if (!strncmp(dbuf.mv_data, "VERSION=", STRLENOF("VERSION="))) { - version=atoi(dbuf.mv_data+STRLENOF("VERSION=")); + version=atoi((char *)dbuf.mv_data+STRLENOF("VERSION=")); if (version > 3) { fprintf(stderr, "%s: line %zd: unsupported VERSION %d\n", prog, lineno, version); @@ -74,9 +74,9 @@ static void readhdr() } else if (!strncmp(dbuf.mv_data, "HEADER=END", STRLENOF("HEADER=END"))) { break; } else if (!strncmp(dbuf.mv_data, "format=", STRLENOF("format="))) { - if (!strncmp(dbuf.mv_data+STRLENOF("FORMAT="), "print", STRLENOF("print"))) + if (!strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "print", STRLENOF("print"))) mode |= PRINT; - else if (strncmp(dbuf.mv_data+STRLENOF("FORMAT="), "bytevalue", STRLENOF("bytevalue"))) { + else if (strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "bytevalue", STRLENOF("bytevalue"))) { fprintf(stderr, "%s: line %zd: unsupported FORMAT %s\n", prog, lineno, (char *)dbuf.mv_data+STRLENOF("FORMAT=")); exit(EXIT_FAILURE); @@ -85,9 +85,9 @@ static void readhdr() ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size); if (ptr) *ptr = '\0'; if (subname) free(subname); - subname = strdup(dbuf.mv_data+STRLENOF("database=")); + subname = strdup((char *)dbuf.mv_data+STRLENOF("database=")); } else if (!strncmp(dbuf.mv_data, "type=", STRLENOF("type="))) { - if (strncmp(dbuf.mv_data+STRLENOF("type="), "btree", STRLENOF("btree"))) { + if (strncmp((char *)dbuf.mv_data+STRLENOF("type="), "btree", STRLENOF("btree"))) { fprintf(stderr, "%s: line %zd: unsupported type %s\n", prog, lineno, (char *)dbuf.mv_data+STRLENOF("type=")); exit(EXIT_FAILURE); From d9423e13f85bac7c33083a4b872c022d18a41232 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Sat, 21 Jun 2014 15:53:41 -0700 Subject: [PATCH 10/16] More cleanup for page_merge, page_loosen Only loosen src page if it's actually dirty. Refactor page refs in page_merge. --- libraries/liblmdb/mdb.c | 49 ++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index c6985b009b..e5bfc84996 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -7251,14 +7251,17 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst) static int mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst) { - int rc; - indx_t i, j; - MDB_node *srcnode; + MDB_page *psrc, *pdst; + MDB_node *srcnode; MDB_val key, data; - unsigned nkeys; + unsigned nkeys; + int rc; + indx_t i, j; - DPRINTF(("merging page %"Z"u into %"Z"u", csrc->mc_pg[csrc->mc_top]->mp_pgno, - cdst->mc_pg[cdst->mc_top]->mp_pgno)); + psrc = csrc->mc_pg[csrc->mc_top]; + pdst = cdst->mc_pg[cdst->mc_top]; + + DPRINTF(("merging page %"Z"u into %"Z"u", psrc->mp_pgno, pdst->mp_pgno)); mdb_cassert(csrc, csrc->mc_snum > 1); /* can't merge root page */ mdb_cassert(csrc, cdst->mc_snum > 1); @@ -7269,20 +7272,20 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst) /* Move all nodes from src to dst. */ - j = nkeys = NUMKEYS(cdst->mc_pg[cdst->mc_top]); - if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { + j = nkeys = NUMKEYS(pdst); + if (IS_LEAF2(psrc)) { key.mv_size = csrc->mc_db->md_pad; - key.mv_data = METADATA(csrc->mc_pg[csrc->mc_top]); - for (i = 0; i < NUMKEYS(csrc->mc_pg[csrc->mc_top]); i++, j++) { + key.mv_data = METADATA(psrc); + for (i = 0; i < NUMKEYS(psrc); i++, j++) { rc = mdb_node_add(cdst, j, &key, NULL, 0, 0); if (rc != MDB_SUCCESS) return rc; key.mv_data = (char *)key.mv_data + key.mv_size; } } else { - for (i = 0; i < NUMKEYS(csrc->mc_pg[csrc->mc_top]); i++, j++) { - srcnode = NODEPTR(csrc->mc_pg[csrc->mc_top], i); - if (i == 0 && IS_BRANCH(csrc->mc_pg[csrc->mc_top])) { + for (i = 0; i < NUMKEYS(psrc); i++, j++) { + srcnode = NODEPTR(psrc, i); + if (i == 0 && IS_BRANCH(psrc)) { MDB_cursor mn; MDB_node *s2; mdb_cursor_copy(csrc, &mn); @@ -7312,8 +7315,8 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst) } DPRINTF(("dst page %"Z"u now has %u keys (%.1f%% filled)", - cdst->mc_pg[cdst->mc_top]->mp_pgno, NUMKEYS(cdst->mc_pg[cdst->mc_top]), - (float)PAGEFILL(cdst->mc_txn->mt_env, cdst->mc_pg[cdst->mc_top]) / 10)); + pdst->mp_pgno, NUMKEYS(pdst), + (float)PAGEFILL(cdst->mc_txn->mt_env, pdst) / 10)); /* Unlink the src page from parent and add to free list. */ @@ -7329,8 +7332,15 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst) } csrc->mc_top++; - mdb_page_loose(csrc->mc_txn->mt_env, csrc->mc_pg[csrc->mc_top]); - if (IS_LEAF(csrc->mc_pg[csrc->mc_top])) + psrc = csrc->mc_pg[csrc->mc_top]; + if (psrc->mp_flags & P_DIRTY) { + mdb_page_loose(csrc->mc_txn->mt_env, psrc); + } else { + rc = mdb_midl_append(&csrc->mc_txn->mt_free_pgs, psrc->mp_pgno); + if (rc) + return rc; + } + if (IS_LEAF(psrc)) csrc->mc_db->md_leaf_pages--; else csrc->mc_db->md_branch_pages--; @@ -7338,7 +7348,6 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst) /* Adjust other cursors pointing to mp */ MDB_cursor *m2, *m3; MDB_dbi dbi = csrc->mc_dbi; - MDB_page *mp = cdst->mc_pg[cdst->mc_top]; for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { if (csrc->mc_flags & C_SUB) @@ -7347,8 +7356,8 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst) m3 = m2; if (m3 == csrc) continue; if (m3->mc_snum < csrc->mc_snum) continue; - if (m3->mc_pg[csrc->mc_top] == csrc->mc_pg[csrc->mc_top]) { - m3->mc_pg[csrc->mc_top] = mp; + if (m3->mc_pg[csrc->mc_top] == psrc) { + m3->mc_pg[csrc->mc_top] = pdst; m3->mc_ki[csrc->mc_top] += nkeys; } } From aa89ca31b3a95f266cd2b354037466a2d950bc1f Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Sun, 22 Jun 2014 16:39:14 -0700 Subject: [PATCH 11/16] More for page_loosen Don't try this when working on the FreeDB. Again, too much hassle to unkink the recursions... --- libraries/liblmdb/mdb.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index e5bfc84996..efd15e5bea 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -7333,7 +7333,10 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst) csrc->mc_top++; psrc = csrc->mc_pg[csrc->mc_top]; - if (psrc->mp_flags & P_DIRTY) { + /* If not operating on FreeDB, allow this page to be reused + * in this txn. + */ + if ((psrc->mp_flags & P_DIRTY) && csrc->mc_dbi != FREE_DBI) { mdb_page_loose(csrc->mc_txn->mt_env, psrc); } else { rc = mdb_midl_append(&csrc->mc_txn->mt_free_pgs, psrc->mp_pgno); From 225bcae3e7bf944d0740609dd331e08900ae4161 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Mon, 23 Jun 2014 07:36:38 -0700 Subject: [PATCH 12/16] Flesh out mdb_cmp_cint for BigEndians --- libraries/liblmdb/mdb.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index efd15e5bea..d2e4b2ca17 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -4610,7 +4610,16 @@ mdb_cmp_cint(const MDB_val *a, const MDB_val *b) } while(!x && u > (unsigned short *)a->mv_data); return x; #else - return memcmp(a->mv_data, b->mv_data, a->mv_size); + unsigned short *u, *c, *end; + int x; + + end = (unsigned short *) ((char *) a->mv_data + a->mv_size); + u = (unsigned short *)a->mv_data; + c = (unsigned short *)b->mv_data; + do { + x = *u++ - *c++; + } while(!x && u < end); + return x; #endif } From ca47c2af1f4d90e1a0ec7e4c6ce4937262c18c74 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Mon, 23 Jun 2014 07:39:06 -0700 Subject: [PATCH 13/16] ITS#7879 Windows build compatibility Just use srand/rand, we don't care about the quality of the random numbers, we just want some data. --- libraries/liblmdb/.gitignore | 1 + libraries/liblmdb/mtest.c | 11 +++++------ libraries/liblmdb/mtest2.c | 11 +++++------ libraries/liblmdb/mtest3.c | 11 +++++------ libraries/liblmdb/mtest4.c | 5 ++--- libraries/liblmdb/mtest5.c | 11 +++++------ libraries/liblmdb/mtest6.c | 7 +++---- 7 files changed, 26 insertions(+), 31 deletions(-) diff --git a/libraries/liblmdb/.gitignore b/libraries/liblmdb/.gitignore index 0d493fe188..0b4b1cba67 100644 --- a/libraries/liblmdb/.gitignore +++ b/libraries/liblmdb/.gitignore @@ -5,6 +5,7 @@ mdb_copy mdb_stat *.[ao] *.so +*.exe *[~#] *.bak *.orig diff --git a/libraries/liblmdb/mtest.c b/libraries/liblmdb/mtest.c index 01579950b1..79b4175e79 100644 --- a/libraries/liblmdb/mtest.c +++ b/libraries/liblmdb/mtest.c @@ -1,6 +1,6 @@ /* mtest.c - memory-mapped database tester/toy */ /* - * Copyright 2011 Howard Chu, Symas Corp. + * Copyright 2011-2014 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -11,7 +11,6 @@ * top-level directory of the distribution or, alternatively, at * . */ -#define _XOPEN_SOURCE 500 /* srandom(), random() */ #include #include #include @@ -36,13 +35,13 @@ int main(int argc,char * argv[]) int *values; char sval[32] = ""; - srandom(time(NULL)); + srand(time(NULL)); - count = (random()%384) + 64; + count = (rand()%384) + 64; values = (int *)malloc(count*sizeof(int)); for(i = 0;i -1; i-= (random()%5)) { + for (i= count - 1; i > -1; i-= (rand()%5)) { j++; txn=NULL; E(mdb_txn_begin(env, NULL, 0, &txn)); diff --git a/libraries/liblmdb/mtest2.c b/libraries/liblmdb/mtest2.c index ebda8521e4..f1a3dbd6c8 100644 --- a/libraries/liblmdb/mtest2.c +++ b/libraries/liblmdb/mtest2.c @@ -1,6 +1,6 @@ /* mtest2.c - memory-mapped database tester/toy */ /* - * Copyright 2011 Howard Chu, Symas Corp. + * Copyright 2011-2014 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -14,7 +14,6 @@ /* Just like mtest.c, but using a subDB instead of the main DB */ -#define _XOPEN_SOURCE 500 /* srandom(), random() */ #include #include #include @@ -38,13 +37,13 @@ int main(int argc,char * argv[]) int *values; char sval[32] = ""; - srandom(time(NULL)); + srand(time(NULL)); - count = (random()%384) + 64; + count = (rand()%384) + 64; values = (int *)malloc(count*sizeof(int)); for(i = 0;i -1; i-= (random()%5)) { + for (i= count - 1; i > -1; i-= (rand()%5)) { j++; txn=NULL; E(mdb_txn_begin(env, NULL, 0, &txn)); diff --git a/libraries/liblmdb/mtest3.c b/libraries/liblmdb/mtest3.c index 95b1749688..f705c52dec 100644 --- a/libraries/liblmdb/mtest3.c +++ b/libraries/liblmdb/mtest3.c @@ -1,6 +1,6 @@ /* mtest3.c - memory-mapped database tester/toy */ /* - * Copyright 2011 Howard Chu, Symas Corp. + * Copyright 2011-2014 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -13,7 +13,6 @@ */ /* Tests for sorted duplicate DBs */ -#define _XOPEN_SOURCE 500 /* srandom(), random() */ #include #include #include @@ -39,15 +38,15 @@ int main(int argc,char * argv[]) char sval[32]; char kval[sizeof(int)]; - srandom(time(NULL)); + srand(time(NULL)); memset(sval, 0, sizeof(sval)); - count = (random()%384) + 64; + count = (rand()%384) + 64; values = (int *)malloc(count*sizeof(int)); for(i = 0;i -1; i-= (random()%5)) { + for (i= count - 1; i > -1; i-= (rand()%5)) { j++; txn=NULL; E(mdb_txn_begin(env, NULL, 0, &txn)); diff --git a/libraries/liblmdb/mtest4.c b/libraries/liblmdb/mtest4.c index 37f95baa24..da5a953044 100644 --- a/libraries/liblmdb/mtest4.c +++ b/libraries/liblmdb/mtest4.c @@ -1,6 +1,6 @@ /* mtest4.c - memory-mapped database tester/toy */ /* - * Copyright 2011 Howard Chu, Symas Corp. + * Copyright 2011-2014 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -13,7 +13,6 @@ */ /* Tests for sorted duplicate DBs with fixed-size keys */ -#define _XOPEN_SOURCE 500 /* srandom(), random() */ #include #include #include @@ -123,7 +122,7 @@ int main(int argc,char * argv[]) mdb_txn_abort(txn); j=0; - for (i= count - 1; i > -1; i-= (random()%3)) { + for (i= count - 1; i > -1; i-= (rand()%3)) { j++; txn=NULL; E(mdb_txn_begin(env, NULL, 0, &txn)); diff --git a/libraries/liblmdb/mtest5.c b/libraries/liblmdb/mtest5.c index 4edfea0d4c..39a8c728ac 100644 --- a/libraries/liblmdb/mtest5.c +++ b/libraries/liblmdb/mtest5.c @@ -1,6 +1,6 @@ /* mtest5.c - memory-mapped database tester/toy */ /* - * Copyright 2011 Howard Chu, Symas Corp. + * Copyright 2011-2014 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -13,7 +13,6 @@ */ /* Tests for sorted duplicate DBs using cursor_put */ -#define _XOPEN_SOURCE 500 /* srandom(), random() */ #include #include #include @@ -39,15 +38,15 @@ int main(int argc,char * argv[]) char sval[32]; char kval[sizeof(int)]; - srandom(time(NULL)); + srand(time(NULL)); memset(sval, 0, sizeof(sval)); - count = (random()%384) + 64; + count = (rand()%384) + 64; values = (int *)malloc(count*sizeof(int)); for(i = 0;i -1; i-= (random()%5)) { + for (i= count - 1; i > -1; i-= (rand()%5)) { j++; txn=NULL; E(mdb_txn_begin(env, NULL, 0, &txn)); diff --git a/libraries/liblmdb/mtest6.c b/libraries/liblmdb/mtest6.c index 8d32e8851a..07d5758932 100644 --- a/libraries/liblmdb/mtest6.c +++ b/libraries/liblmdb/mtest6.c @@ -1,6 +1,6 @@ /* mtest6.c - memory-mapped database tester/toy */ /* - * Copyright 2011 Howard Chu, Symas Corp. + * Copyright 2011-2014 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -13,7 +13,6 @@ */ /* Tests for DB splits and merges */ -#define _XOPEN_SOURCE 500 /* srandom(), random() */ #include #include #include @@ -41,7 +40,7 @@ int main(int argc,char * argv[]) long kval; char *sval; - srandom(time(NULL)); + srand(time(NULL)); E(mdb_env_create(&env)); E(mdb_env_set_mapsize(env, 10485760)); @@ -90,7 +89,7 @@ int main(int argc,char * argv[]) #if 0 j=0; - for (i= count - 1; i > -1; i-= (random()%5)) { + for (i= count - 1; i > -1; i-= (rand()%5)) { j++; txn=NULL; E(mdb_txn_begin(env, NULL, 0, &txn)); From 9a4ef8406eac173875e67131421a684c34bf92f3 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Tue, 24 Jun 2014 04:16:19 -0700 Subject: [PATCH 14/16] Doc updates Rename MDB -> LMDB Integrate tool manpages --- libraries/liblmdb/Doxyfile | 4 +-- libraries/liblmdb/lmdb.h | 58 ++++++++++++++++++++++---------------- libraries/liblmdb/tooltag | 22 +++++++++++++++ 3 files changed, 57 insertions(+), 27 deletions(-) create mode 100644 libraries/liblmdb/tooltag diff --git a/libraries/liblmdb/Doxyfile b/libraries/liblmdb/Doxyfile index 3fd0365c7d..92d17b09eb 100644 --- a/libraries/liblmdb/Doxyfile +++ b/libraries/liblmdb/Doxyfile @@ -25,7 +25,7 @@ DOXYFILE_ENCODING = UTF-8 # The PROJECT_NAME tag is a single word (or a sequence of words surrounded # by quotes) that should identify the project. -PROJECT_NAME = MDB +PROJECT_NAME = LMDB # The PROJECT_NUMBER tag can be used to enter a project or revision number. # This could be handy for archiving the generated documentation or @@ -1404,7 +1404,7 @@ SKIP_FUNCTION_MACROS = YES # If a tag file is not located in the directory in which doxygen # is run, you must also specify the path to the tagfile here. -TAGFILES = +TAGFILES = tooltag=./man1 # When a file name is specified after GENERATE_TAGFILE, doxygen will create # a tag file that is based on the input files it reads. diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h index c7a259049b..98d9cc1e2c 100644 --- a/libraries/liblmdb/lmdb.h +++ b/libraries/liblmdb/lmdb.h @@ -1,10 +1,10 @@ /** @file lmdb.h * @brief Lightning memory-mapped database library * - * @mainpage Lightning Memory-Mapped Database Manager (MDB) + * @mainpage Lightning Memory-Mapped Database Manager (LMDB) * * @section intro_sec Introduction - * MDB is a Btree-based database management library modeled loosely on the + * LMDB is a Btree-based database management library modeled loosely on the * BerkeleyDB API, but much simplified. The entire database is exposed * in a memory map, and all data fetches return data directly * from the mapped memory, so no malloc's or memcpy's occur during @@ -26,10 +26,10 @@ * readers, and readers don't block writers. * * Unlike other well-known database mechanisms which use either write-ahead - * transaction logs or append-only data writes, MDB requires no maintenance + * transaction logs or append-only data writes, LMDB requires no maintenance * during operation. Both write-ahead loggers and append-only databases * require periodic checkpointing and/or compaction of their log or database - * files otherwise they grow without bound. MDB tracks free pages within + * files otherwise they grow without bound. LMDB tracks free pages within * the database and re-uses them for new write operations, so the database * size does not grow without bound in normal use. * @@ -49,7 +49,7 @@ * stale locks can block further operation. * * Fix: Check for stale readers periodically, using the - * #mdb_reader_check function or the mdb_stat tool. Or just + * #mdb_reader_check function or the \ref mdb_stat_1 "mdb_stat" tool. Or just * make all programs using the database close it; the lockfile * is always reset on first open of the environment. * @@ -86,7 +86,7 @@ * * - Use an MDB_env* in the process which opened it, without fork()ing. * - * - Do not have open an MDB database twice in the same process at + * - Do not have open an LMDB database twice in the same process at * the same time. Not even from a plain open() call - close()ing it * breaks flock() advisory locking. * @@ -109,7 +109,7 @@ * - If you do that anyway, do a periodic check for stale readers. Or * close the environment once in a while, so the lockfile can get reset. * - * - Do not use MDB databases on remote filesystems, even between + * - Do not use LMDB databases on remote filesystems, even between * processes on the same host. This breaks flock() on some OSes, * possibly memory map sync, and certainly sync between programs * on different hosts. @@ -172,7 +172,7 @@ typedef void *mdb_filehandle_t; typedef int mdb_filehandle_t; #endif -/** @defgroup mdb MDB API +/** @defgroup mdb LMDB API * @{ * @brief OpenLDAP Lightning Memory-Mapped Database Manager */ @@ -386,7 +386,7 @@ typedef enum MDB_cursor_op { #define MDB_PANIC (-30795) /** Environment version mismatch */ #define MDB_VERSION_MISMATCH (-30794) - /** File is not a valid MDB file */ + /** File is not a valid LMDB file */ #define MDB_INVALID (-30793) /** Environment mapsize reached */ #define MDB_MAP_FULL (-30792) @@ -436,7 +436,7 @@ typedef struct MDB_envinfo { unsigned int me_numreaders; /**< max reader slots used in the environment */ } MDB_envinfo; - /** @brief Return the mdb library version information. + /** @brief Return the LMDB library version information. * * @param[out] major if non-NULL, the library major version number is copied here * @param[out] minor if non-NULL, the library minor version number is copied here @@ -450,14 +450,14 @@ char *mdb_version(int *major, int *minor, int *patch); * This function is a superset of the ANSI C X3.159-1989 (ANSI C) strerror(3) * function. If the error code is greater than or equal to 0, then the string * returned by the system function strerror(3) is returned. If the error code - * is less than 0, an error string corresponding to the MDB library error is - * returned. See @ref errors for a list of MDB-specific error codes. + * is less than 0, an error string corresponding to the LMDB library error is + * returned. See @ref errors for a list of LMDB-specific error codes. * @param[in] err The error code * @retval "error message" The description of the error */ char *mdb_strerror(int err); - /** @brief Create an MDB environment handle. + /** @brief Create an LMDB environment handle. * * This function allocates memory for a #MDB_env structure. To release * the allocated memory and discard the handle, call #mdb_env_close(). @@ -490,15 +490,15 @@ int mdb_env_create(MDB_env **env); * how the operating system has allocated memory to shared libraries and other uses. * The feature is highly experimental. *
  • #MDB_NOSUBDIR - * By default, MDB creates its environment in a directory whose + * By default, LMDB creates its environment in a directory whose * pathname is given in \b path, and creates its data and lock files * under that directory. With this option, \b path is used as-is for * the database main data file. The database lock file is the \b path * with "-lock" appended. *
  • #MDB_RDONLY * Open the environment in read-only mode. No write operations will be - * allowed. MDB will still modify the lock file - except on read-only - * filesystems, where MDB does not use locks. + * allowed. LMDB will still modify the lock file - except on read-only + * filesystems, where LMDB does not use locks. *
  • #MDB_WRITEMAP * Use a writeable memory map unless MDB_RDONLY is set. This is faster * and uses fewer mallocs, but loses protection from application bugs @@ -542,7 +542,7 @@ int mdb_env_create(MDB_env **env); * the user synchronizes its use. Applications that multiplex many * user threads over individual OS threads need this option. Such an * application must also serialize the write transactions in an OS - * thread, since MDB's write locking is unaware of the user threads. + * thread, since LMDB's write locking is unaware of the user threads. *
  • #MDB_NOLOCK * Don't do any locking. If concurrent access is anticipated, the * caller must manage all concurrency itself. For proper operation @@ -581,7 +581,7 @@ int mdb_env_create(MDB_env **env); * @return A non-zero error value on failure and 0 on success. Some possible * errors are: *
      - *
    • #MDB_VERSION_MISMATCH - the version of the MDB library doesn't match the + *
    • #MDB_VERSION_MISMATCH - the version of the LMDB library doesn't match the * version that created the database environment. *
    • #MDB_INVALID - the environment file headers are corrupted. *
    • ENOENT - the directory specified by the path parameter doesn't exist. @@ -591,7 +591,7 @@ int mdb_env_create(MDB_env **env); */ int mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode); - /** @brief Copy an MDB environment to the specified path. + /** @brief Copy an LMDB environment to the specified path. * * This function may be used to make a backup of an existing environment. * No lockfile is created, since it gets recreated at need. @@ -607,7 +607,7 @@ int mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t */ int mdb_env_copy(MDB_env *env, const char *path); - /** @brief Copy an MDB environment to the specified file descriptor. + /** @brief Copy an LMDB environment to the specified file descriptor. * * This function may be used to make a backup of an existing environment. * No lockfile is created, since it gets recreated at need. @@ -622,7 +622,7 @@ int mdb_env_copy(MDB_env *env, const char *path); */ int mdb_env_copyfd(MDB_env *env, mdb_filehandle_t fd); - /** @brief Return statistics about the MDB environment. + /** @brief Return statistics about the LMDB environment. * * @param[in] env An environment handle returned by #mdb_env_create() * @param[out] stat The address of an #MDB_stat structure @@ -630,7 +630,7 @@ int mdb_env_copyfd(MDB_env *env, mdb_filehandle_t fd); */ int mdb_env_stat(MDB_env *env, MDB_stat *stat); - /** @brief Return information about the MDB environment. + /** @brief Return information about the LMDB environment. * * @param[in] env An environment handle returned by #mdb_env_create() * @param[out] stat The address of an #MDB_envinfo structure @@ -641,7 +641,7 @@ int mdb_env_info(MDB_env *env, MDB_envinfo *stat); /** @brief Flush the data buffers to disk. * * Data is always written to disk when #mdb_txn_commit() is called, - * but the operating system may keep it buffered. MDB always flushes + * but the operating system may keep it buffered. LMDB always flushes * the OS buffers upon commit as well, unless the environment was * opened with #MDB_NOSYNC or in part #MDB_NOMETASYNC. * @param[in] env An environment handle returned by #mdb_env_create() @@ -824,7 +824,7 @@ int mdb_env_set_userctx(MDB_env *env, void *ctx); */ void *mdb_env_get_userctx(MDB_env *env); - /** @brief A callback function for most MDB assert() failures, + /** @brief A callback function for most LMDB assert() failures, * called before printing the message and aborting. * * @param[in] env An environment handle returned by #mdb_env_create(). @@ -1206,7 +1206,7 @@ int mdb_get(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data); * reserved space, which the caller can fill in later - before * the next update operation or the transaction ends. This saves * an extra memcpy if the data is being generated later. - * MDB does nothing else with this memory, the caller is expected + * LMDB does nothing else with this memory, the caller is expected * to modify all of the space requested. *
    • #MDB_APPEND - append the given key/data pair to the end of the * database. No key comparisons are performed. This option allows @@ -1478,4 +1478,12 @@ int mdb_reader_check(MDB_env *env, int *dead); #ifdef __cplusplus } #endif +/** @page tools LMDB Command Line Tools + The following describes the command line tools that are available for LMDB. + \li \ref mdb_copy_1 + \li \ref mdb_dump_1 + \li \ref mdb_load_1 + \li \ref mdb_stat_1 +*/ + #endif /* _LMDB_H_ */ diff --git a/libraries/liblmdb/tooltag b/libraries/liblmdb/tooltag new file mode 100644 index 0000000000..229bf16baa --- /dev/null +++ b/libraries/liblmdb/tooltag @@ -0,0 +1,22 @@ + + + mdb_copy_1 + mdb_copy - environment copy tool + mdb_copy.1 + + + mdb_dump_1 + mdb_dump - environment export tool + mdb_dump.1 + + + mdb_load_1 + mdb_load - environment import tool + mdb_load.1 + + + mdb_stat_1 + mdb_stat - environment status tool + mdb_stat.1 + + From 7fe85f5c8291f531f93c28b98865367aeca75484 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Tue, 24 Jun 2014 04:42:44 -0700 Subject: [PATCH 15/16] More MDB -> LMDB renames --- libraries/liblmdb/mdb.c | 18 +++++++++--------- libraries/liblmdb/midl.c | 2 +- libraries/liblmdb/midl.h | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index d2e4b2ca17..c98247c2b9 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -1,5 +1,5 @@ /** @file mdb.c - * @brief memory-mapped database library + * @brief Lightning memory-mapped database library * * A Btree-based database management library modeled loosely on the * BerkeleyDB API, but much simplified. @@ -145,7 +145,7 @@ # error "Two's complement, reasonably sized integer types, please" #endif -/** @defgroup internal MDB Internals +/** @defgroup internal LMDB Internals * @{ */ /** @defgroup compat Compatibility Macros @@ -381,7 +381,7 @@ static txnid_t mdb_debug_start; */ #define MDB_MINKEYS 2 - /** A stamp that identifies a file as an MDB file. + /** A stamp that identifies a file as an LMDB file. * There's nothing special about this value other than that it is easily * recognizable, and it will reflect any byte order mismatches. */ @@ -568,7 +568,7 @@ typedef struct MDB_reader { * unlikely. If a collision occurs, the results are unpredictable. */ typedef struct MDB_txbody { - /** Stamp identifying this as an MDB file. It must be set + /** Stamp identifying this as an LMDB file. It must be set * to #MDB_MAGIC. */ uint32_t mtb_magic; /** Format of this lock file. Must be set to #MDB_LOCK_FORMAT. */ @@ -842,7 +842,7 @@ typedef struct MDB_db { * Pages 0-1 are meta pages. Transaction N writes meta page #(N % 2). */ typedef struct MDB_meta { - /** Stamp identifying this as an MDB file. It must be set + /** Stamp identifying this as an LMDB file. It must be set * to #MDB_MAGIC. */ uint32_t mm_magic; /** Version number of this lock file. Must be set to #MDB_DATA_VERSION. */ @@ -1185,7 +1185,7 @@ mdb_version(int *major, int *minor, int *patch) return MDB_VERSION_STRING; } -/** Table of descriptions for MDB @ref errors */ +/** Table of descriptions for LMDB @ref errors */ static char *const mdb_errstr[] = { "MDB_KEYEXIST: Key/data pair already exists", "MDB_NOTFOUND: No matching key/data pair found", @@ -1193,7 +1193,7 @@ static char *const mdb_errstr[] = { "MDB_CORRUPTED: Located page was wrong type", "MDB_PANIC: Update of meta page failed", "MDB_VERSION_MISMATCH: Database environment version mismatch", - "MDB_INVALID: File is not an MDB file", + "MDB_INVALID: File is not an LMDB file", "MDB_MAP_FULL: Environment mapsize limit reached", "MDB_DBS_FULL: Environment maxdbs limit reached", "MDB_READERS_FULL: Environment maxreaders limit reached", @@ -3623,7 +3623,7 @@ mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers) return MDB_SUCCESS; } -/** Further setup required for opening an MDB environment +/** Further setup required for opening an LMDB environment */ static int mdb_env_open2(MDB_env *env) @@ -3959,7 +3959,7 @@ mdb_hash_enc(MDB_val *val, char *encbuf) #endif /** Open and/or initialize the lock region for the environment. - * @param[in] env The MDB environment. + * @param[in] env The LMDB environment. * @param[in] lpath The pathname of the file used for the lock region. * @param[in] mode The Unix permissions for the file, if we create it. * @param[out] excl Resulting file lock type: -1 none, 0 shared, 1 exclusive diff --git a/libraries/liblmdb/midl.c b/libraries/liblmdb/midl.c index b39d463c56..d3d872a29b 100644 --- a/libraries/liblmdb/midl.c +++ b/libraries/liblmdb/midl.c @@ -22,7 +22,7 @@ #include #include "midl.h" -/** @defgroup internal MDB Internals +/** @defgroup internal LMDB Internals * @{ */ /** @defgroup idls ID List Management diff --git a/libraries/liblmdb/midl.h b/libraries/liblmdb/midl.h index f2bb4338de..08616f452e 100644 --- a/libraries/liblmdb/midl.h +++ b/libraries/liblmdb/midl.h @@ -1,5 +1,5 @@ /** @file midl.h - * @brief mdb ID List header file. + * @brief LMDB ID List header file. * * This file was originally part of back-bdb but has been * modified for use in libmdb. Most of the macros defined @@ -32,7 +32,7 @@ extern "C" { #endif -/** @defgroup internal MDB Internals +/** @defgroup internal LMDB Internals * @{ */ From a3e5539505ce6397b052078deff5ed816fe2ad16 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Tue, 24 Jun 2014 11:43:13 -0700 Subject: [PATCH 16/16] More for mdb_page_loose Require that the page is dirty in the current txn, not a parent txn. --- libraries/liblmdb/mdb.c | 81 +++++++++++++++++++++++++++++------------ 1 file changed, 58 insertions(+), 23 deletions(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index c98247c2b9..c21f556e08 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -899,6 +899,10 @@ struct MDB_txn { /** The list of pages that became unused during this transaction. */ MDB_IDL mt_free_pgs; + /** The list of loose pages that became unused and may be reused + * in this transaction. + */ + MDB_page *mt_loose_pgs; /** The sorted list of dirty pages we temporarily wrote to disk * because the dirty list was full. page numbers in here are * shifted left by 1, deleted slots have the LSB set. @@ -1022,7 +1026,6 @@ typedef struct MDB_xcursor { typedef struct MDB_pgstate { pgno_t *mf_pghead; /**< Reclaimed freeDB pages, or NULL before use */ txnid_t mf_pglast; /**< ID of last used record, or 0 if !mf_pghead */ - MDB_page *mf_pgloose; /**< Dirty pages that can be reused */ } MDB_pgstate; /** The database environment. */ @@ -1059,7 +1062,6 @@ struct MDB_env { MDB_pgstate me_pgstate; /**< state of old pages from freeDB */ # define me_pglast me_pgstate.mf_pglast # define me_pghead me_pgstate.mf_pghead -# define me_pgloose me_pgstate.mf_pgloose MDB_page *me_dpages; /**< list of malloc'd blocks for re-use */ /** IDL of pages that became unused in a write txn */ MDB_IDL me_free_pgs; @@ -1527,21 +1529,58 @@ mdb_dlist_free(MDB_txn *txn) dl[0].mid = 0; } -/** Loosen a single page. +/** Loosen or free a single page. * Saves single pages to a list for future reuse * in this same txn. It has been pulled from the freeDB * and already resides on the dirty list, but has been * deleted. Use these pages first before pulling again * from the freeDB. + * + * If the page wasn't dirtied in this txn, just add it + * to this txn's free list. */ -static void -mdb_page_loose(MDB_env *env, MDB_page *mp) +static int +mdb_page_loose(MDB_cursor *mc, MDB_page *mp) { + int loose = 0; + pgno_t pgno = mp->mp_pgno; + + if ((mp->mp_flags & P_DIRTY) && mc->mc_dbi != FREE_DBI) { + if (mc->mc_txn->mt_parent) { + MDB_ID2 *dl = mc->mc_txn->mt_u.dirty_list; + /* If txn has a parent, make sure the page is in our + * dirty list. + */ + if (dl[0].mid) { + unsigned x = mdb_mid2l_search(dl, pgno); + if (x <= dl[0].mid && dl[x].mid == pgno) { + if (mp != dl[x].mptr) { /* bad cursor? */ + mc->mc_flags &= ~(C_INITIALIZED|C_EOF); + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return MDB_CORRUPTED; + } + /* ok, it's ours */ + loose = 1; + } + } + } else { + /* no parent txn, so it's just ours */ + loose = 1; + } + } + if (loose) { pgno_t *pp = (pgno_t *)mp->mp_ptrs; - *pp = mp->mp_pgno; - mp->mp_next = env->me_pgloose; - env->me_pgloose = mp; + *pp = pgno; + mp->mp_next = mc->mc_txn->mt_loose_pgs; + mc->mc_txn->mt_loose_pgs = mp; mp->mp_flags |= P_LOOSE; + } else { + int rc = mdb_midl_append(&mc->mc_txn->mt_free_pgs, pgno); + if (rc) + return rc; + } + + return MDB_SUCCESS; } /** Set or clear P_KEEP in dirty, non-overflow, non-sub pages watched by txn. @@ -1593,7 +1632,7 @@ mdb_pages_xkeep(MDB_cursor *mc, unsigned pflags, int all) } /* Loose pages shouldn't be spilled */ - for (dp = txn->mt_env->me_pgloose; dp; dp=dp->mp_next) { + for (dp = txn->mt_loose_pgs; dp; dp=dp->mp_next) { if ((dp->mp_flags & Mask) == pflags) dp->mp_flags ^= P_KEEP; } @@ -1826,10 +1865,10 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) MDB_cursor m2; /* If there are any loose pages, just use them */ - if (num == 1 && env->me_pgloose) { + if (num == 1 && txn->mt_loose_pgs) { pgno_t *pp; - np = env->me_pgloose; - env->me_pgloose = np->mp_next; + np = txn->mt_loose_pgs; + txn->mt_loose_pgs = np->mp_next; pp = (pgno_t *)np->mp_ptrs; np->mp_pgno = *pp; *mp = np; @@ -2700,8 +2739,8 @@ mdb_freelist_save(MDB_txn *txn) /* Dispose of loose pages. Usually they will have all * been used up by the time we get here. */ - if (env->me_pgloose) { - MDB_page *mp = env->me_pgloose; + if (txn->mt_loose_pgs) { + MDB_page *mp = txn->mt_loose_pgs; pgno_t *pp; /* Just return them to freeDB */ if (env->me_pghead) { @@ -2726,7 +2765,7 @@ mdb_freelist_save(MDB_txn *txn) mp = mp->mp_next; } } - env->me_pgloose = NULL; + txn->mt_loose_pgs = NULL; } /* MDB_RESERVE cancels meminit in ovpage malloc (when no WRITEMAP) */ @@ -7343,15 +7382,11 @@ mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst) psrc = csrc->mc_pg[csrc->mc_top]; /* If not operating on FreeDB, allow this page to be reused - * in this txn. + * in this txn. Otherwise just add to free list. */ - if ((psrc->mp_flags & P_DIRTY) && csrc->mc_dbi != FREE_DBI) { - mdb_page_loose(csrc->mc_txn->mt_env, psrc); - } else { - rc = mdb_midl_append(&csrc->mc_txn->mt_free_pgs, psrc->mp_pgno); - if (rc) - return rc; - } + rc = mdb_page_loose(csrc, psrc); + if (rc) + return rc; if (IS_LEAF(psrc)) csrc->mc_db->md_leaf_pages--; else