Merge remote-tracking branch 'origin/mdb.RE/0.9'

libraries/liblmdb/CHANGES

@@ -2,8 +2,12 @@ LMDB 0.9 Change Log
 
 LMDB 0.9.18 Release Engineering
 	Fix robust mutex detection on glibc 2.10-11 (ITS#8330)
+	Check for utf8_to_utf16 failures (ITS#7992)
+	Catch strdup failure in mdb_dbi_open
 	Build
 		Additional makefile var tweaks (ITS#8169)
+	Documentation
+		Add Getting Started page
 	
 
 LMDB 0.9.17 Release (2015/11/30)

libraries/liblmdb/Doxyfile

@@ -582,7 +582,7 @@ WARN_LOGFILE           =
 # directories like "/usr/src/myproject". Separate the files or directories
 # with spaces.
 
-INPUT                  = lmdb.h midl.h mdb.c midl.c
+INPUT                  = lmdb.h midl.h mdb.c midl.c intro.doc
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is

libraries/liblmdb/intro.doc

@@ -0,0 +1,192 @@
+/*
+ * Copyright 2015 Howard Chu, Symas Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * <http://www.OpenLDAP.org/license.html>.
+ */
+/** @page starting Getting Started
+
+LMDB is compact, fast, powerful, and robust and implements a simplified
+variant of the BerkeleyDB (BDB) API. (BDB is also very powerful, and verbosely
+documented in its own right.) After reading this page, the main
+\ref mdb documentation should make sense. Thanks to Bert Hubert
+for creating the
+<a href="https://github.com/ahupowerdns/ahutils/blob/master/lmdb-semantics.md">
+initial version</a> of this writeup.
+
+Everything starts with an environment, created by #mdb_env_create().
+Once created, this environment must also be opened with #mdb_env_open().
+
+#mdb_env_open() gets passed a name which is interpreted as a directory
+path. Note that this directory must exist already, it is not created
+for you. Within that directory, a lock file and a storage file will be
+generated. If you don't want to use a directory, you can pass the
+#MDB_NOSUBDIR option, in which case the path you provided is used
+directly as the data file, and another file with a "-lock" suffix
+added will be used for the lock file.
+
+Once the environment is open, a transaction can be created within it
+using #mdb_txn_begin(). Transactions may be read-write or read-only,
+and read-write transactions may be nested. A transaction must only
+be used by one thread at a time. Transactions are always required,
+even for read-only access. The transaction provides a consistent
+view of the data.
+
+Once a transaction has been created, a database can be opened within it
+using #mdb_dbi_open(). If only one database will ever be used in the
+environment, a NULL can be passed as the database name. For named
+databases, the #MDB_CREATE flag must be used to create the database
+if it doesn't already exist. Also, #mdb_env_set_maxdbs() must be
+called after #mdb_env_create() and before #mdb_env_open() to set the
+maximum number of named databases you want to support.
+
+Note: a single transaction can open multiple databases. Generally
+databases should only be opened once, by the first transaction in
+the process. After the first transaction completes, the database
+handles can freely be used by all subsequent transactions.
+
+Within a transaction, #mdb_get() and #mdb_put() can store single
+key/value pairs if that is all you need to do (but see \ref Cursors
+below if you want to do more).
+
+A key/value pair is expressed as two #MDB_val structures. This struct
+has two fields, \c mv_size and \c mv_data. The data is a \c void pointer to
+an array of \c mv_size bytes.
+
+Because LMDB is very efficient (and usually zero-copy), the data returned
+in an #MDB_val structure may be memory-mapped straight from disk. In
+other words <b>look but do not touch</b> (or free() for that matter).
+Once a transaction is closed, the values can no longer be used, so
+make a copy if you need to keep them after that.
+
+@section Cursors Cursors
+
+To do more powerful things, we must use a cursor.
+
+Within the transaction, a cursor can be created with #mdb_cursor_open().
+With this cursor we can store/retrieve/delete (multiple) values using
+#mdb_cursor_get(), #mdb_cursor_put(), and #mdb_cursor_del().
+
+#mdb_cursor_get() positions itself depending on the cursor operation
+requested, and for some operations, on the supplied key. For example,
+to list all key/value pairs in a database, use operation #MDB_FIRST for
+the first call to #mdb_cursor_get(), and #MDB_NEXT on subsequent calls,
+until the end is hit.
+
+To retrieve all keys starting from a specified key value, use #MDB_SET.
+For more cursor operations, see the \ref mdb docs.
+
+When using #mdb_cursor_put(), either the function will position the
+cursor for you based on the \b key, or you can use operation
+#MDB_CURRENT to use the current position of the cursor. Note that
+\b key must then match the current position's key.
+
+@subsection summary Summarizing the Opening
+
+So we have a cursor in a transaction which opened a database in an
+environment which is opened from a filesystem after it was
+separately created.
+
+Or, we create an environment, open it from a filesystem, create a
+transaction within it, open a database within that transaction,
+and create a cursor within all of the above.
+
+Got it?
+
+@section thrproc Threads and Processes
+
+LMDB uses POSIX locks on files, and these locks have issues if one
+process opens a file multiple times. Because of this, do not
+#mdb_env_open() a file multiple times from a single process. Instead,
+share the LMDB environment that has opened the file across all threads.
+Otherwise, if a single process opens the same environment multiple times,
+closing it once will remove all the locks held on it, and the other
+instances will be vulnerable to corruption from other processes.
+
+Also note that a transaction is tied to one thread by default using
+Thread Local Storage. If you want to pass read-only transactions across
+threads, you can use the #MDB_NOTLS option on the environment.
+
+@section txns Transactions, Rollbacks, etc.
+
+To actually get anything done, a transaction must be committed using
+#mdb_txn_commit(). Alternatively, all of a transaction's operations
+can be discarded using #mdb_txn_abort(). In a read-only transaction,
+any cursors will \b not automatically be freed. In a read-write
+transaction, all cursors will be freed and must not be used again.
+
+For read-only transactions, obviously there is nothing to commit to
+storage. The transaction still must eventually be aborted to close
+any database handle(s) opened in it, or committed to keep the
+database handles around for reuse in new transactions.
+
+In addition, as long as a transaction is open, a consistent view of
+the database is kept alive, which requires storage. A read-only
+transaction that no longer requires this consistent view should
+be terminated (committed or aborted) when the view is no longer
+needed (but see below for an optimization).
+
+There can be multiple simultaneously active read-only transactions
+but only one that can write. Once a single read-write transaction
+is opened, all further attempts to begin one will block until the
+first one is committed or aborted. This has no effect on read-only
+transactions, however, and they may continue to be opened at any time.
+
+@section dupkeys Duplicate Keys
+
+#mdb_get() and #mdb_put() respectively have no and only some support
+for multiple key/value pairs with identical keys. If there are multiple
+values for a key, #mdb_get() will only return the first value.
+
+When multiple values for one key are required, pass the #MDB_DUPSORT
+flag to #mdb_dbi_open(). In an #MDB_DUPSORT database, by default
+#mdb_put() will not replace the value for a key if the key existed
+already. Instead it will add the new value to the key. In addition,
+#mdb_del() will pay attention to the value field too, allowing for
+specific values of a key to be deleted.
+
+Finally, additional cursor operations become available for
+traversing through and retrieving duplicate values.
+
+@section optim Some Optimization
+
+If you frequently begin and abort read-only transactions, as an
+optimization, it is possible to only reset and renew a transaction.
+
+#mdb_txn_reset() releases any old copies of data kept around for
+a read-only transaction. To reuse this reset transaction, call
+#mdb_txn_renew() on it. Any cursors in this transaction must also
+be renewed using #mdb_cursor_renew().
+
+Note that #mdb_txn_reset() is similar to #mdb_txn_abort() and will
+close any databases you opened within the transaction.
+
+To permanently free a transaction, reset or not, use #mdb_txn_abort().
+
+@section cleanup Cleaning Up
+
+For read-only transactions, any cursors created within it must
+be closed using #mdb_cursor_close().
+
+It is very rarely necessary to close a database handle, and in
+general they should just be left open.
+
+@section onward The Full API
+
+The full \ref mdb documentation lists further details, like how to:
+
+  \li size a database (the default limits are intentionally small)
+  \li drop and clean a database
+  \li detect and report errors
+  \li optimize (bulk) loading speed
+  \li (temporarily) reduce robustness to gain even more speed
+  \li gather statistics about the database
+  \li define custom sort orders
+
+*/

libraries/liblmdb/lmdb.h

@@ -40,6 +40,9 @@
  *	corrupt the database. Of course if your application code is known to
  *	be bug-free (...) then this is not an issue.
  *
+ *	If this is your first time using a transactional embedded key/value
+ *	store, you may find the \ref starting page to be helpful.
+ *
  *	@section caveats_sec Caveats
  *	Troubleshooting the lock file, plus semaphores on BSD systems:
  *
@@ -191,7 +194,7 @@ typedef int mdb_filehandle_t;
 /** Library minor version */
 #define MDB_VERSION_MINOR	9
 /** Library patch version */
-#define MDB_VERSION_PATCH	17
+#define MDB_VERSION_PATCH	18
 
 /** Combine args a,b,c into a single integer for easy version comparisons */
 #define MDB_VERINT(a,b,c)	(((a) << 24) | ((b) << 16) | (c))
@@ -201,7 +204,7 @@ typedef int mdb_filehandle_t;
 	MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH)
 
 /** The release date of this library version */
-#define MDB_VERSION_DATE	"November 30, 2015"
+#define MDB_VERSION_DATE	"December 19, 2015"
 
 /** A stringifier for the version info */
 #define MDB_VERSTR(a,b,c,d)	"LMDB " #a "." #b "." #c ": (" d ")"

libraries/liblmdb/mdb.c

@@ -4470,7 +4470,9 @@ mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl)
 
 #ifdef _WIN32
 	wchar_t *wlpath;
-	utf8_to_utf16(lpath, -1, &wlpath, NULL);
+	rc = utf8_to_utf16(lpath, -1, &wlpath, NULL);
+	if (rc)
+		return rc;
 	env->me_lfd = CreateFileW(wlpath, GENERIC_READ|GENERIC_WRITE,
 		FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, OPEN_ALWAYS,
 		FILE_ATTRIBUTE_NORMAL, NULL);
@@ -4758,7 +4760,9 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 		len = OPEN_ALWAYS;
 	}
 	mode = FILE_ATTRIBUTE_NORMAL;
-	utf8_to_utf16(dpath, -1, &wpath, NULL);
+	rc = utf8_to_utf16(dpath, -1, &wpath, NULL);
+	if (rc)
+		goto leave;
 	env->me_fd = CreateFileW(wpath, oflags, FILE_SHARE_READ|FILE_SHARE_WRITE,
 		NULL, len, mode, NULL);
 	free(wpath);
@@ -4790,7 +4794,9 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 			 */
 #ifdef _WIN32
 			len = OPEN_EXISTING;
-			utf8_to_utf16(dpath, -1, &wpath, NULL);
+			rc = utf8_to_utf16(dpath, -1, &wpath, NULL);
+			if (rc)
+				goto leave;
 			env->me_mfd = CreateFileW(wpath, oflags,
 				FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, len,
 				mode | FILE_FLAG_WRITE_THROUGH, NULL);
@@ -9281,7 +9287,9 @@ mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags)
 	 * already in the OS cache.
 	 */
 #ifdef _WIN32
-	utf8_to_utf16(lpath, -1, &wpath, NULL);
+	rc = utf8_to_utf16(lpath, -1, &wpath, NULL);
+	if (rc)
+		return rc;
 	newfd = CreateFileW(wpath, GENERIC_WRITE, 0, NULL, CREATE_NEW,
 				FILE_FLAG_NO_BUFFERING|FILE_FLAG_WRITE_THROUGH, NULL);
 	free(wpath);
@@ -9476,6 +9484,7 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db
 	MDB_db dummy;
 	int rc, dbflag, exact;
 	unsigned int unused = 0, seq;
+	char *namedup;
 	size_t len;
 
 	if (flags & ~VALID_FLAGS)
@@ -9537,8 +9546,16 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db
 		MDB_node *node = NODEPTR(mc.mc_pg[mc.mc_top], mc.mc_ki[mc.mc_top]);
 		if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) != F_SUBDATA)
 			return MDB_INCOMPATIBLE;
-	} else if (rc == MDB_NOTFOUND && (flags & MDB_CREATE)) {
-		/* Create if requested */
+	} else if (! (rc == MDB_NOTFOUND && (flags & MDB_CREATE))) {
+		return rc;
+	}
+
+	/* Done here so we cannot fail after creating a new DB */
+	if ((namedup = strdup(name)) == NULL)
+		return ENOMEM;
+
+	if (rc) {
+		/* MDB_NOTFOUND and MDB_CREATE: Create new DB */
 		data.mv_size = sizeof(MDB_db);
 		data.mv_data = &dummy;
 		memset(&dummy, 0, sizeof(dummy));
@@ -9548,10 +9565,12 @@ int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *db
 		dbflag |= DB_DIRTY;
 	}
 
-	/* OK, got info, add to table */
-	if (rc == MDB_SUCCESS) {
+	if (rc) {
+		free(namedup);
+	} else {
+		/* Got info, register DBI in this txn */
 		unsigned int slot = unused ? unused : txn->mt_numdbs;
-		txn->mt_dbxs[slot].md_name.mv_data = strdup(name);
+		txn->mt_dbxs[slot].md_name.mv_data = namedup;
 		txn->mt_dbxs[slot].md_name.mv_size = len;
 		txn->mt_dbxs[slot].md_rel = NULL;
 		txn->mt_dbflags[slot] = dbflag;
@@ -10002,6 +10021,8 @@ static int utf8_to_utf16(const char *src, int srcsize, wchar_t **dst, int *dstsi
 	if (need == 0)
 		return EINVAL;
 	result = malloc(sizeof(wchar_t) * need);
+	if (!result)
+		return ENOMEM;
 	MultiByteToWideChar(CP_UTF8, 0, src, srcsize, result, need);
 	if (dstsize)
 		*dstsize = need;