From 18caeaa70b1aaa376b85ebc2a9be6f1bc222deed Mon Sep 17 00:00:00 2001 From: Hallvard Furuseth Date: Sat, 12 Dec 2015 19:25:06 +0100 Subject: [PATCH 1/4] mdb_dbi_open(): Catch strdup failure --- libraries/liblmdb/mdb.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index f495d6c1f8..36968efff2 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -9476,6 +9476,7 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db MDB_db dummy; int rc, dbflag, exact; unsigned int unused = 0, seq; + char *namedup; size_t len; if (flags & ~VALID_FLAGS) @@ -9537,8 +9538,16 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db MDB_node *node = NODEPTR(mc.mc_pg[mc.mc_top], mc.mc_ki[mc.mc_top]); if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) != F_SUBDATA) return MDB_INCOMPATIBLE; - } else if (rc == MDB_NOTFOUND && (flags & MDB_CREATE)) { - /* Create if requested */ + } else if (! (rc == MDB_NOTFOUND && (flags & MDB_CREATE))) { + return rc; + } + + /* Done here so we cannot fail after creating a new DB */ + if ((namedup = strdup(name)) == NULL) + return ENOMEM; + + if (rc) { + /* MDB_NOTFOUND and MDB_CREATE: Create new DB */ data.mv_size = sizeof(MDB_db); data.mv_data = &dummy; memset(&dummy, 0, sizeof(dummy)); @@ -9548,10 +9557,12 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db dbflag |= DB_DIRTY; } - /* OK, got info, add to table */ - if (rc == MDB_SUCCESS) { + if (rc) { + free(namedup); + } else { + /* Got info, register DBI in this txn */ unsigned int slot = unused ? unused : txn->mt_numdbs; - txn->mt_dbxs[slot].md_name.mv_data = strdup(name); + txn->mt_dbxs[slot].md_name.mv_data = namedup; txn->mt_dbxs[slot].md_name.mv_size = len; txn->mt_dbxs[slot].md_rel = NULL; txn->mt_dbflags[slot] = dbflag; From ec32e90022f43af86db5330ed5a28a38644f1692 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Tue, 15 Dec 2015 18:45:34 +0000 Subject: [PATCH 2/4] ITS#7992 cleanup check for utf8_to_utf16 failures --- libraries/liblmdb/mdb.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 36968efff2..fa0c9e5b9c 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -4470,7 +4470,9 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl) #ifdef _WIN32 wchar_t *wlpath; - utf8_to_utf16(lpath, -1, &wlpath, NULL); + rc = utf8_to_utf16(lpath, -1, &wlpath, NULL); + if (rc) + return rc; env->me_lfd = CreateFileW(wlpath, GENERIC_READ|GENERIC_WRITE, FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); @@ -4758,7 +4760,9 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode len = OPEN_ALWAYS; } mode = FILE_ATTRIBUTE_NORMAL; - utf8_to_utf16(dpath, -1, &wpath, NULL); + rc = utf8_to_utf16(dpath, -1, &wpath, NULL); + if (rc) + goto leave; env->me_fd = CreateFileW(wpath, oflags, FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, len, mode, NULL); free(wpath); @@ -4790,7 +4794,9 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode */ #ifdef _WIN32 len = OPEN_EXISTING; - utf8_to_utf16(dpath, -1, &wpath, NULL); + rc = utf8_to_utf16(dpath, -1, &wpath, NULL); + if (rc) + goto leave; env->me_mfd = CreateFileW(wpath, oflags, FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, len, mode | FILE_FLAG_WRITE_THROUGH, NULL); @@ -9281,7 +9287,9 @@ mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags) * already in the OS cache. */ #ifdef _WIN32 - utf8_to_utf16(lpath, -1, &wpath, NULL); + rc = utf8_to_utf16(lpath, -1, &wpath, NULL); + if (rc) + return rc; newfd = CreateFileW(wpath, GENERIC_WRITE, 0, NULL, CREATE_NEW, FILE_FLAG_NO_BUFFERING|FILE_FLAG_WRITE_THROUGH, NULL); free(wpath); @@ -10013,6 +10021,8 @@ static int utf8_to_utf16(const char *src, int srcsize, wchar_t **dst, int *dstsi if (need == 0) return EINVAL; result = malloc(sizeof(wchar_t) * need); + if (!result) + return ENOMEM; MultiByteToWideChar(CP_UTF8, 0, src, srcsize, result, need); if (dstsize) *dstsize = need; From 3fd0d5fb80089729594af089dff11ade66e00375 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Sat, 19 Dec 2015 22:53:26 +0000 Subject: [PATCH 3/4] Add Getting Started doc --- libraries/liblmdb/Doxyfile | 2 +- libraries/liblmdb/intro.doc | 192 ++++++++++++++++++++++++++++++++++++ libraries/liblmdb/lmdb.h | 3 + 3 files changed, 196 insertions(+), 1 deletion(-) create mode 100644 libraries/liblmdb/intro.doc diff --git a/libraries/liblmdb/Doxyfile b/libraries/liblmdb/Doxyfile index 92d17b09eb..5047c0bb1f 100644 --- a/libraries/liblmdb/Doxyfile +++ b/libraries/liblmdb/Doxyfile @@ -582,7 +582,7 @@ WARN_LOGFILE = # directories like "/usr/src/myproject". Separate the files or directories # with spaces. -INPUT = lmdb.h midl.h mdb.c midl.c +INPUT = lmdb.h midl.h mdb.c midl.c intro.doc # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is diff --git a/libraries/liblmdb/intro.doc b/libraries/liblmdb/intro.doc new file mode 100644 index 0000000000..870c7bb8e7 --- /dev/null +++ b/libraries/liblmdb/intro.doc @@ -0,0 +1,192 @@ +/* + * Copyright 2015 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ +/** @page starting Getting Started + +LMDB is compact, fast, powerful, and robust and implements a simplified +variant of the BerkeleyDB (BDB) API. (BDB is also very powerful, and verbosely +documented in its own right.) After reading this page, the main +\ref mdb documentation should make sense. Thanks to Bert Hubert +for creating the + +initial version of this writeup. + +Everything starts with an environment, created by #mdb_env_create(). +Once created, this environment must also be opened with #mdb_env_open(). + +#mdb_env_open() gets passed a name which is interpreted as a directory +path. Note that this directory must exist already, it is not created +for you. Within that directory, a lock file and a storage file will be +generated. If you don't want to use a directory, you can pass the +#MDB_NOSUBDIR option, in which case the path you provided is used +directly as the data file, and another file with a "-lock" suffix +added will be used for the lock file. + +Once the environment is open, a transaction can be created within it +using #mdb_txn_begin(). Transactions may be read-write or read-only, +and read-write transactions may be nested. A transaction must only +be used by one thread at a time. Transactions are always required, +even for read-only access. The transaction provides a consistent +view of the data. + +Once a transaction has been created, a database can be opened within it +using #mdb_dbi_open(). If only one database will ever be used in the +environment, a NULL can be passed as the database name. For named +databases, the #MDB_CREATE flag must be used to create the database +if it doesn't already exist. Also, #mdb_env_set_maxdbs() must be +called after #mdb_env_create() and before #mdb_env_open() to set the +maximum number of named databases you want to support. + +Note: a single transaction can open multiple databases. Generally +databases should only be opened once, by the first transaction in +the process. After the first transaction completes, the database +handles can freely be used by all subsequent transactions. + +Within a transaction, #mdb_get() and #mdb_put() can store single +key/value pairs if that is all you need to do (but see \ref Cursors +below if you want to do more). + +A key/value pair is expressed as two #MDB_val structures. This struct +has two fields, \c mv_size and \c mv_data. The data is a \c void pointer to +an array of \c mv_size bytes. + +Because LMDB is very efficient (and usually zero-copy), the data returned +in an #MDB_val structure may be memory-mapped straight from disk. In +other words look but do not touch (or free() for that matter). +Once a transaction is closed, the values can no longer be used, so +make a copy if you need to keep them after that. + +@section Cursors Cursors + +To do more powerful things, we must use a cursor. + +Within the transaction, a cursor can be created with #mdb_cursor_open(). +With this cursor we can store/retrieve/delete (multiple) values using +#mdb_cursor_get(), #mdb_cursor_put(), and #mdb_cursor_del(). + +#mdb_cursor_get() positions itself depending on the cursor operation +requested, and for some operations, on the supplied key. For example, +to list all key/value pairs in a database, use operation #MDB_FIRST for +the first call to #mdb_cursor_get(), and #MDB_NEXT on subsequent calls, +until the end is hit. + +To retrieve all keys starting from a specified key value, use #MDB_SET. +For more cursor operations, see the \ref mdb docs. + +When using #mdb_cursor_put(), either the function will position the +cursor for you based on the \b key, or you can use operation +#MDB_CURRENT to use the current position of the cursor. Note that +\b key must then match the current position's key. + +@subsection summary Summarizing the Opening + +So we have a cursor in a transaction which opened a database in an +environment which is opened from a filesystem after it was +separately created. + +Or, we create an environment, open it from a filesystem, create a +transaction within it, open a database within that transaction, +and create a cursor within all of the above. + +Got it? + +@section thrproc Threads and Processes + +LMDB uses POSIX locks on files, and these locks have issues if one +process opens a file multiple times. Because of this, do not +#mdb_env_open() a file multiple times from a single process. Instead, +share the LMDB environment that has opened the file across all threads. +Otherwise, if a single process opens the same environment multiple times, +closing it once will remove all the locks held on it, and the other +instances will be vulnerable to corruption from other processes. + +Also note that a transaction is tied to one thread by default using +Thread Local Storage. If you want to pass read-only transactions across +threads, you can use the #MDB_NOTLS option on the environment. + +@section txns Transactions, Rollbacks, etc. + +To actually get anything done, a transaction must be committed using +#mdb_txn_commit(). Alternatively, all of a transaction's operations +can be discarded using #mdb_txn_abort(). In a read-only transaction, +any cursors will \b not automatically be freed. In a read-write +transaction, all cursors will be freed and must not be used again. + +For read-only transactions, obviously there is nothing to commit to +storage. The transaction still must eventually be aborted to close +any database handle(s) opened in it, or committed to keep the +database handles around for reuse in new transactions. + +In addition, as long as a transaction is open, a consistent view of +the database is kept alive, which requires storage. A read-only +transaction that no longer requires this consistent view should +be terminated (committed or aborted) when the view is no longer +needed (but see below for an optimization). + +There can be multiple simultaneously active read-only transactions +but only one that can write. Once a single read-write transaction +is opened, all further attempts to begin one will block until the +first one is committed or aborted. This has no effect on read-only +transactions, however, and they may continue to be opened at any time. + +@section dupkeys Duplicate Keys + +#mdb_get() and #mdb_put() respectively have no and only some support +for multiple key/value pairs with identical keys. If there are multiple +values for a key, #mdb_get() will only return the first value. + +When multiple values for one key are required, pass the #MDB_DUPSORT +flag to #mdb_dbi_open(). In an #MDB_DUPSORT database, by default +#mdb_put() will not replace the value for a key if the key existed +already. Instead it will add the new value to the key. In addition, +#mdb_del() will pay attention to the value field too, allowing for +specific values of a key to be deleted. + +Finally, additional cursor operations become available for +traversing through and retrieving duplicate values. + +@section optim Some Optimization + +If you frequently begin and abort read-only transactions, as an +optimization, it is possible to only reset and renew a transaction. + +#mdb_txn_reset() releases any old copies of data kept around for +a read-only transaction. To reuse this reset transaction, call +#mdb_txn_renew() on it. Any cursors in this transaction must also +be renewed using #mdb_cursor_renew(). + +Note that #mdb_txn_reset() is similar to #mdb_txn_abort() and will +close any databases you opened within the transaction. + +To permanently free a transaction, reset or not, use #mdb_txn_abort(). + +@section cleanup Cleaning Up + +For read-only transactions, any cursors created within it must +be closed using #mdb_cursor_close(). + +It is very rarely necessary to close a database handle, and in +general they should just be left open. + +@section onward The Full API + +The full \ref mdb documentation lists further details, like how to: + + \li size a database (the default limits are intentionally small) + \li drop and clean a database + \li detect and report errors + \li optimize (bulk) loading speed + \li (temporarily) reduce robustness to gain even more speed + \li gather statistics about the database + \li define custom sort orders + +*/ diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h index fa7d62c57a..20bfc0e722 100644 --- a/libraries/liblmdb/lmdb.h +++ b/libraries/liblmdb/lmdb.h @@ -40,6 +40,9 @@ * corrupt the database. Of course if your application code is known to * be bug-free (...) then this is not an issue. * + * If this is your first time using a transactional embedded key/value + * store, you may find the \ref starting page to be helpful. + * * @section caveats_sec Caveats * Troubleshooting the lock file, plus semaphores on BSD systems: * From e8760b474d3a2a4ae78dd7dc7a17698ac5425575 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Sat, 19 Dec 2015 23:06:00 +0000 Subject: [PATCH 4/4] ITS#7992 cleanup, new docs, mdb_dbi_open cleanup --- libraries/liblmdb/CHANGES | 4 ++++ libraries/liblmdb/lmdb.h | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/libraries/liblmdb/CHANGES b/libraries/liblmdb/CHANGES index 4665e8666f..5bfb22ef7a 100644 --- a/libraries/liblmdb/CHANGES +++ b/libraries/liblmdb/CHANGES @@ -2,8 +2,12 @@ LMDB 0.9 Change Log LMDB 0.9.18 Release Engineering Fix robust mutex detection on glibc 2.10-11 (ITS#8330) + Check for utf8_to_utf16 failures (ITS#7992) + Catch strdup failure in mdb_dbi_open Build Additional makefile var tweaks (ITS#8169) + Documentation + Add Getting Started page LMDB 0.9.17 Release (2015/11/30) diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h index 20bfc0e722..3ecdc1063c 100644 --- a/libraries/liblmdb/lmdb.h +++ b/libraries/liblmdb/lmdb.h @@ -194,7 +194,7 @@ typedef int mdb_filehandle_t; /** Library minor version */ #define MDB_VERSION_MINOR 9 /** Library patch version */ -#define MDB_VERSION_PATCH 17 +#define MDB_VERSION_PATCH 18 /** Combine args a,b,c into a single integer for easy version comparisons */ #define MDB_VERINT(a,b,c) (((a) << 24) | ((b) << 16) | (c)) @@ -204,7 +204,7 @@ typedef int mdb_filehandle_t; MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH) /** The release date of this library version */ -#define MDB_VERSION_DATE "November 30, 2015" +#define MDB_VERSION_DATE "December 19, 2015" /** A stringifier for the version info */ #define MDB_VERSTR(a,b,c,d) "LMDB " #a "." #b "." #c ": (" d ")"