Merge remote-tracking branch 'origin/mdb.master'

This commit is contained in:
Quanah Gibson-Mount 2014-07-08 17:21:10 -05:00
commit bfd8b3a42a
9 changed files with 936 additions and 302 deletions

View file

@ -10,6 +10,9 @@ mdb_stat
*.bak
*.orig
*.rej
*.gcov
*.gcda
*.gcno
core
core.*
valgrind.*

View file

@ -17,7 +17,7 @@
# read mdb.c before changing any of them.
#
CC = gcc
W = -W -Wall -Wno-unused-parameter -Wbad-function-cast
W = -W -Wall -Wno-unused-parameter -Wbad-function-cast -Wuninitialized
THREADS = -pthread
OPT = -O2 -g
CFLAGS = $(THREADS) $(OPT) $(W) $(XCFLAGS)
@ -76,3 +76,22 @@ midl.o: midl.c midl.h
%.o: %.c lmdb.h
$(CC) $(CFLAGS) $(CPPFLAGS) -c $<
COV_FLAGS=-fprofile-arcs -ftest-coverage
COV_OBJS=xmdb.o xmidl.o
coverage: xmtest
for i in mtest*.c [0-9]*.c; do j=`basename \$$i .c`; $(MAKE) $$j.o; \
gcc -o x$$j $$j.o $(COV_OBJS) -pthread $(COV_FLAGS); \
rm -rf testdb; mkdir testdb; ./x$$j; done
gcov xmdb.c
gcov xmidl.c
xmtest: mtest.o xmdb.o xmidl.o
gcc -o xmtest mtest.o xmdb.o xmidl.o -pthread $(COV_FLAGS)
xmdb.o: mdb.c lmdb.h midl.h
$(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -O0 $(COV_FLAGS) -c mdb.c -o $@
xmidl.o: midl.c midl.h
$(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -O0 $(COV_FLAGS) -c midl.c -o $@

View file

@ -333,6 +333,15 @@ typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *rel
#define MDB_MULTIPLE 0x80000
/* @} */
/** @defgroup mdb_copy Copy Flags
* @{
*/
/** Compacting copy: Omit free space from copy, and renumber all
* pages sequentially.
*/
#define MDB_CP_COMPACT 0x01
/* @} */
/** @brief Cursor Get operations.
*
* This is the set of all operations for retrieving data
@ -622,6 +631,49 @@ int mdb_env_copy(MDB_env *env, const char *path);
*/
int mdb_env_copyfd(MDB_env *env, mdb_filehandle_t fd);
/** @brief Copy an LMDB environment to the specified path, with options.
*
* This function may be used to make a backup of an existing environment.
* No lockfile is created, since it gets recreated at need.
* @note This call can trigger significant file size growth if run in
* parallel with write transactions, because it employs a read-only
* transaction. See long-lived transactions under @ref caveats_sec.
* @param[in] env An environment handle returned by #mdb_env_create(). It
* must have already been opened successfully.
* @param[in] path The directory in which the copy will reside. This
* directory must already exist and be writable but must otherwise be
* empty.
* @param[in] flags Special options for this operation. This parameter
* must be set to 0 or by bitwise OR'ing together one or more of the
* values described here.
* <ul>
* <li>#MDB_CP_COMPACT - Perform compaction while copying: omit free
* pages and sequentially renumber all pages in output. This option
* consumes more CPU and runs more slowly than the default.
* </ul>
* @return A non-zero error value on failure and 0 on success.
*/
int mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags);
/** @brief Copy an LMDB environment to the specified file descriptor,
* with options.
*
* This function may be used to make a backup of an existing environment.
* No lockfile is created, since it gets recreated at need. See
* #mdb_env_copy2() for further details.
* @note This call can trigger significant file size growth if run in
* parallel with write transactions, because it employs a read-only
* transaction. See long-lived transactions under @ref caveats_sec.
* @param[in] env An environment handle returned by #mdb_env_create(). It
* must have already been opened successfully.
* @param[in] fd The filedescriptor to write the copy to. It must
* have already been opened for Write access.
* @param[in] flags Special options for this operation.
* See #mdb_env_copy2() for options.
* @return A non-zero error value on failure and 0 on success.
*/
int mdb_env_copyfd2(MDB_env *env, mdb_filehandle_t fd, unsigned int flags);
/** @brief Return statistics about the LMDB environment.
*
* @param[in] env An environment handle returned by #mdb_env_create()

File diff suppressed because it is too large Load diff

View file

@ -8,6 +8,8 @@ mdb_copy \- LMDB environment copy tool
[\c
.BR \-V ]
[\c
.BR \-c ]
[\c
.BR \-n ]
.B srcpath
[\c
@ -30,6 +32,11 @@ written to stdout.
.BR \-V
Write the library version number to the standard output, and exit.
.TP
.BR \-c
Compact while copying. Only current data pages will be copied; freed
or unused pages will be omitted from the copy. This option will
slow down the backup process as it is more CPU-intensive.
.TP
.BR \-n
Open LDMB environment(s) which do not use subdirectories.

View file

@ -33,10 +33,13 @@ int main(int argc,char * argv[])
MDB_env *env;
const char *progname = argv[0], *act;
unsigned flags = MDB_RDONLY;
unsigned cpflags = 0;
for (; argc > 1 && argv[1][0] == '-'; argc--, argv++) {
if (argv[1][1] == 'n' && argv[1][2] == '\0')
flags |= MDB_NOSUBDIR;
else if (argv[1][1] == 'c' && argv[1][2] == '\0')
cpflags |= MDB_CP_COMPACT;
else if (argv[1][1] == 'V' && argv[1][2] == '\0') {
printf("%s\n", MDB_VERSION_STRING);
exit(0);
@ -45,7 +48,7 @@ int main(int argc,char * argv[])
}
if (argc<2 || argc>3) {
fprintf(stderr, "usage: %s [-V] [-n] srcpath [dstpath]\n", progname);
fprintf(stderr, "usage: %s [-V] [-c] [-n] srcpath [dstpath]\n", progname);
exit(EXIT_FAILURE);
}
@ -66,9 +69,9 @@ int main(int argc,char * argv[])
if (rc == MDB_SUCCESS) {
act = "copying";
if (argc == 2)
rc = mdb_env_copyfd(env, MDB_STDOUT);
rc = mdb_env_copyfd2(env, MDB_STDOUT, cpflags);
else
rc = mdb_env_copy(env, argv[2]);
rc = mdb_env_copy2(env, argv[2], cpflags);
}
if (rc)
fprintf(stderr, "%s: %s failed, error %d (%s)\n",

View file

@ -17,8 +17,15 @@
#include <string.h>
#include <ctype.h>
#include <unistd.h>
#include <signal.h>
#include "lmdb.h"
#ifdef _WIN32
#define Z "I"
#else
#define Z "z"
#endif
#define PRINT 1
static int mode;
@ -37,6 +44,13 @@ flagbit dbflags[] = {
{ 0, NULL }
};
static volatile sig_atomic_t gotsig;
static void dumpsig( int sig )
{
gotsig=1;
}
static const char hexc[] = "0123456789abcdef";
static void hex(unsigned char c)
@ -83,6 +97,7 @@ static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name)
MDB_cursor *mc;
MDB_stat ms;
MDB_val key, data;
MDB_envinfo info;
unsigned int flags;
int rc, i;
@ -92,11 +107,18 @@ static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name)
rc = mdb_stat(txn, dbi, &ms);
if (rc) return rc;
rc = mdb_env_info(mdb_txn_env(txn), &info);
if (rc) return rc;
printf("VERSION=3\n");
printf("format=%s\n", mode & PRINT ? "print" : "bytevalue");
if (name)
printf("database=%s\n", name);
printf("type=btree\n");
printf("mapsize=%" Z "u\n", info.me_mapsize);
if (info.me_mapaddr)
printf("mapaddr=%p\n", info.me_mapaddr);
printf("maxreaders=%u\n", info.me_maxreaders);
if (flags & MDB_DUPSORT)
printf("duplicates=1\n");
@ -112,6 +134,10 @@ static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name)
if (rc) return rc;
while ((rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT) == MDB_SUCCESS)) {
if (gotsig) {
rc = EINTR;
break;
}
if (mode & PRINT) {
text(&key);
text(&data);
@ -196,6 +222,15 @@ int main(int argc, char *argv[])
if (optind != argc - 1)
usage(prog);
#ifdef SIGPIPE
signal(SIGPIPE, dumpsig);
#endif
#ifdef SIGHUP
signal(SIGHUP, dumpsig);
#endif
signal(SIGINT, dumpsig);
signal(SIGTERM, dumpsig);
envname = argv[optind];
rc = mdb_env_create(&env);
@ -205,19 +240,19 @@ int main(int argc, char *argv[])
rc = mdb_env_open(env, envname, envflags | MDB_RDONLY, 0664);
if (rc) {
printf("mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
if (rc) {
printf("mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
rc = mdb_open(txn, subname, 0, &dbi);
if (rc) {
printf("mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
@ -228,7 +263,7 @@ int main(int argc, char *argv[])
rc = mdb_cursor_open(txn, dbi, &cursor);
if (rc) {
printf("mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
while ((rc = mdb_cursor_get(cursor, &key, NULL, MDB_NEXT_NODUP)) == 0) {
@ -247,6 +282,8 @@ int main(int argc, char *argv[])
list++;
} else {
rc = dumpit(txn, db2, str);
if (rc)
break;
}
mdb_close(env, db2);
}

View file

@ -32,10 +32,18 @@ static int flags;
static char *prog;
static int eof;
static int Eof;
static MDB_envinfo info;
static MDB_val kbuf, dbuf;
#ifdef _WIN32
#define Z "I"
#else
#define Z "z"
#endif
#define STRLENOF(s) (sizeof(s)-1)
typedef struct flagbit {
@ -67,7 +75,7 @@ static void readhdr()
if (!strncmp(dbuf.mv_data, "VERSION=", STRLENOF("VERSION="))) {
version=atoi((char *)dbuf.mv_data+STRLENOF("VERSION="));
if (version > 3) {
fprintf(stderr, "%s: line %zd: unsupported VERSION %d\n",
fprintf(stderr, "%s: line %" Z "d: unsupported VERSION %d\n",
prog, lineno, version);
exit(EXIT_FAILURE);
}
@ -77,7 +85,7 @@ static void readhdr()
if (!strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "print", STRLENOF("print")))
mode |= PRINT;
else if (strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "bytevalue", STRLENOF("bytevalue"))) {
fprintf(stderr, "%s: line %zd: unsupported FORMAT %s\n",
fprintf(stderr, "%s: line %" Z "d: unsupported FORMAT %s\n",
prog, lineno, (char *)dbuf.mv_data+STRLENOF("FORMAT="));
exit(EXIT_FAILURE);
}
@ -88,10 +96,40 @@ static void readhdr()
subname = strdup((char *)dbuf.mv_data+STRLENOF("database="));
} else if (!strncmp(dbuf.mv_data, "type=", STRLENOF("type="))) {
if (strncmp((char *)dbuf.mv_data+STRLENOF("type="), "btree", STRLENOF("btree"))) {
fprintf(stderr, "%s: line %zd: unsupported type %s\n",
fprintf(stderr, "%s: line %" Z "d: unsupported type %s\n",
prog, lineno, (char *)dbuf.mv_data+STRLENOF("type="));
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.mv_data, "mapaddr=", STRLENOF("mapaddr="))) {
int i;
ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
if (ptr) *ptr = '\0';
i = sscanf((char *)dbuf.mv_data+STRLENOF("mapaddr="), "%p", &info.me_mapaddr);
if (i != 1) {
fprintf(stderr, "%s: line %" Z "d: invalid mapaddr %s\n",
prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapaddr="));
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.mv_data, "mapsize=", STRLENOF("mapsize="))) {
int i;
ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
if (ptr) *ptr = '\0';
i = sscanf((char *)dbuf.mv_data+STRLENOF("mapsize="), "%" Z "u", &info.me_mapsize);
if (i != 1) {
fprintf(stderr, "%s: line %" Z "d: invalid mapsize %s\n",
prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapsize="));
exit(EXIT_FAILURE);
}
} else if (!strncmp(dbuf.mv_data, "maxreaders=", STRLENOF("maxreaders="))) {
int i;
ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size);
if (ptr) *ptr = '\0';
i = sscanf((char *)dbuf.mv_data+STRLENOF("maxreaders="), "%u", &info.me_maxreaders);
if (i != 1) {
fprintf(stderr, "%s: line %" Z "d: invalid maxreaders %s\n",
prog, lineno, (char *)dbuf.mv_data+STRLENOF("maxreaders="));
exit(EXIT_FAILURE);
}
} else {
int i;
for (i=0; dbflags[i].bit; i++) {
@ -104,12 +142,12 @@ static void readhdr()
if (!dbflags[i].bit) {
ptr = memchr(dbuf.mv_data, '=', dbuf.mv_size);
if (!ptr) {
fprintf(stderr, "%s: line %zd: unexpected format\n",
fprintf(stderr, "%s: line %" Z "d: unexpected format\n",
prog, lineno);
exit(EXIT_FAILURE);
} else {
*ptr = '\0';
fprintf(stderr, "%s: line %zd: unrecognized keyword ignored: %s\n",
fprintf(stderr, "%s: line %" Z "d: unrecognized keyword ignored: %s\n",
prog, lineno, (char *)dbuf.mv_data);
}
}
@ -119,7 +157,7 @@ static void readhdr()
static void badend()
{
fprintf(stderr, "%s: line %zd: unexpected end of input\n",
fprintf(stderr, "%s: line %" Z "d: unexpected end of input\n",
prog, lineno);
}
@ -146,14 +184,14 @@ static int readline(MDB_val *out, MDB_val *buf)
if (!(mode & NOHDR)) {
c = fgetc(stdin);
if (c == EOF) {
eof = 1;
Eof = 1;
return EOF;
}
if (c != ' ') {
lineno++;
if (fgets(buf->mv_data, buf->mv_size, stdin) == NULL) {
badend:
eof = 1;
Eof = 1;
badend();
return EOF;
}
@ -163,7 +201,7 @@ badend:
}
}
if (fgets(buf->mv_data, buf->mv_size, stdin) == NULL) {
eof = 1;
Eof = 1;
return EOF;
}
lineno++;
@ -175,15 +213,15 @@ badend:
while (c1[len-1] != '\n') {
buf->mv_data = realloc(buf->mv_data, buf->mv_size*2);
if (!buf->mv_data) {
eof = 1;
fprintf(stderr, "%s: line %zd: out of memory, line too long\n",
Eof = 1;
fprintf(stderr, "%s: line %" Z "d: out of memory, line too long\n",
prog, lineno);
return EOF;
}
c1 = buf->mv_data;
c1 += buf->mv_size;
if (fgets((char *)c1, buf->mv_size, stdin) == NULL) {
eof = 1;
Eof = 1;
badend();
return EOF;
}
@ -202,7 +240,7 @@ badend:
c1++; c2 += 2;
} else {
if (c2+3 >= end || !isxdigit(c2[1]) || !isxdigit(c2[2])) {
eof = 1;
Eof = 1;
badend();
return EOF;
}
@ -216,13 +254,13 @@ badend:
} else {
/* odd length not allowed */
if (len & 1) {
eof = 1;
Eof = 1;
badend();
return EOF;
}
while (c2 < end) {
if (!isxdigit(*c2) || !isxdigit(c2[1])) {
eof = 1;
Eof = 1;
badend();
return EOF;
}
@ -251,6 +289,7 @@ int main(int argc, char *argv[])
MDB_dbi dbi;
char *envname;
int envflags = 0, putflags = 0;
int dohdr = 0;
prog = argv[0];
@ -298,45 +337,60 @@ int main(int argc, char *argv[])
if (optind != argc - 1)
usage(prog);
dbuf.mv_size = 4096;
dbuf.mv_data = malloc(dbuf.mv_size);
if (!(mode & NOHDR))
readhdr();
envname = argv[optind];
rc = mdb_env_create(&env);
mdb_env_set_maxdbs(env, 2);
if (info.me_maxreaders)
mdb_env_set_maxreaders(env, info.me_maxreaders);
if (info.me_mapsize)
mdb_env_set_mapsize(env, info.me_mapsize);
if (info.me_mapaddr)
envflags |= MDB_FIXEDMAP;
rc = mdb_env_open(env, envname, envflags, 0664);
if (rc) {
printf("mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
kbuf.mv_size = mdb_env_get_maxkeysize(env) * 2 + 2;
kbuf.mv_data = malloc(kbuf.mv_size);
dbuf.mv_size = 4096;
dbuf.mv_data = malloc(dbuf.mv_size);
while(!eof) {
while(!Eof) {
MDB_val key, data;
int batch = 0;
flags = 0;
if (!(mode & NOHDR))
if (!dohdr) {
dohdr = 1;
} else if (!(mode & NOHDR))
readhdr();
rc = mdb_txn_begin(env, NULL, 0, &txn);
if (rc) {
printf("mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
rc = mdb_open(txn, subname, flags|MDB_CREATE, &dbi);
if (rc) {
printf("mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
rc = mdb_cursor_open(txn, dbi, &mc);
if (rc) {
printf("mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
@ -360,18 +414,18 @@ int main(int argc, char *argv[])
if (batch == 100) {
rc = mdb_txn_commit(txn);
if (rc) {
fprintf(stderr, "%s: line %zd: txn_commit: %s\n",
fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n",
prog, lineno, mdb_strerror(rc));
goto env_close;
}
rc = mdb_txn_begin(env, NULL, 0, &txn);
if (rc) {
printf("mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
rc = mdb_cursor_open(txn, dbi, &mc);
if (rc) {
printf("mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
batch = 0;
@ -380,7 +434,7 @@ int main(int argc, char *argv[])
rc = mdb_txn_commit(txn);
txn = NULL;
if (rc) {
fprintf(stderr, "%s: line %zd: txn_commit: %s\n",
fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n",
prog, lineno, mdb_strerror(rc));
goto env_close;
}

View file

@ -112,7 +112,7 @@ int main(int argc, char *argv[])
rc = mdb_env_open(env, envname, envflags | MDB_RDONLY, 0664);
if (rc) {
printf("mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
@ -145,7 +145,7 @@ int main(int argc, char *argv[])
rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
if (rc) {
printf("mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc));
goto env_close;
}
@ -158,12 +158,12 @@ int main(int argc, char *argv[])
dbi = 0;
rc = mdb_cursor_open(txn, dbi, &cursor);
if (rc) {
printf("mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
rc = mdb_stat(txn, dbi, &mst);
if (rc) {
printf("mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc));
fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
prstat(&mst);
@ -201,13 +201,13 @@ int main(int argc, char *argv[])
rc = mdb_open(txn, subname, 0, &dbi);
if (rc) {
printf("mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
rc = mdb_stat(txn, dbi, &mst);
if (rc) {
printf("mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc));
fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
printf("Status of %s\n", subname ? subname : "Main DB");
@ -219,7 +219,7 @@ int main(int argc, char *argv[])
rc = mdb_cursor_open(txn, dbi, &cursor);
if (rc) {
printf("mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
while ((rc = mdb_cursor_get(cursor, &key, NULL, MDB_NEXT_NODUP)) == 0) {
@ -237,7 +237,7 @@ int main(int argc, char *argv[])
if (rc) continue;
rc = mdb_stat(txn, db2, &mst);
if (rc) {
printf("mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc));
fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc));
goto txn_abort;
}
prstat(&mst);