upstream/openldap
1
0
Fork 0
mirror of https://git.openldap.org/openldap/openldap.git synced 2025-12-21 15:19:34 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

Allow mdb_env_set_mapsize() on an open environment

Browse source 
The caller is responsible for making sure no transactions are
active in this process before resizing. This is slightly lighter
weight than doing a full env_close/env_open cycle.
This commit is contained in:
Howard Chu 2013-08-28 19:12:59 -07:00
parent 2a28686e4f
commit 2e7130cab0
2 changed files with 118 additions and 85 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
libraries/liblmdb/lmdb.h
View file

@ -662,8 +662,15 @@ int mdb_env_get_path(MDB_env *env, const char **path);
* 10485760 bytes. The size of the memory map is also the maximum size * 10485760 bytes. The size of the memory map is also the maximum size
* of the database. The value should be chosen as large as possible, * of the database. The value should be chosen as large as possible,
* to accommodate future growth of the database. * to accommodate future growth of the database.
* This function may only be called after #mdb_env_create() and before #mdb_env_open(). * This function should be called after #mdb_env_create() and before #mdb_env_open().
* The size may be changed by closing and reopening the environment. * It may be called at later times if no transactions are active in
* this process. Note that the library does not check for this condition,
* the caller must ensure it explicitly.
*
* If the mapsize is changed by another process, #mdb_txn_begin() will
* return #MDB_MAP_RESIZED. This function may be called with a size
* of zero to adopt the new size.
*
* Any attempt to set a size smaller than the space already consumed * Any attempt to set a size smaller than the space already consumed
* by the environment will be silently changed to the current size of the used space. * by the environment will be silently changed to the current size of the used space.
* @param[in] env An environment handle returned by #mdb_env_create() * @param[in] env An environment handle returned by #mdb_env_create()
@ -671,7 +678,8 @@ int mdb_env_get_path(MDB_env *env, const char **path);
* @return A non-zero error value on failure and 0 on success. Some possible * @return A non-zero error value on failure and 0 on success. Some possible
* errors are: * errors are:
* <ul> * <ul>
* <li>EINVAL - an invalid parameter was specified, or the environment is already open. * <li>EINVAL - an invalid parameter was specified, or the environment has
* an active write transaction.
* </ul> * </ul>
*/ */
int mdb_env_set_mapsize(MDB_env *env, size_t size); int mdb_env_set_mapsize(MDB_env *env, size_t size);
@ -757,7 +765,8 @@ int mdb_env_get_maxkeysize(MDB_env *env);
* <li>#MDB_PANIC - a fatal error occurred earlier and the environment * <li>#MDB_PANIC - a fatal error occurred earlier and the environment
* must be shut down. * must be shut down.
* <li>#MDB_MAP_RESIZED - another process wrote data beyond this MDB_env's * <li>#MDB_MAP_RESIZED - another process wrote data beyond this MDB_env's
* mapsize and the environment must be shut down. * mapsize and this environment's map must be resized as well.
* See #mdb_env_set_mapsize().
* <li>#MDB_READERS_FULL - a read-only transaction was requested and * <li>#MDB_READERS_FULL - a read-only transaction was requested and
* the reader lock table is full. See #mdb_env_set_maxreaders(). * the reader lock table is full. See #mdb_env_set_maxreaders().
* <li>ENOMEM - out of memory. * <li>ENOMEM - out of memory.

186
libraries/liblmdb/mdb.c
View file

@ -3053,7 +3053,7 @@ mdb_env_init_meta(MDB_env *env, MDB_meta *meta)
DPUTS("writing new meta page"); DPUTS("writing new meta page");
GET_PAGESIZE(psize); psize = env->me_psize;
meta->mm_magic = MDB_MAGIC; meta->mm_magic = MDB_MAGIC;
meta->mm_version = MDB_DATA_VERSION; meta->mm_version = MDB_DATA_VERSION;
@ -3238,11 +3238,97 @@ mdb_env_create(MDB_env **env)
return MDB_SUCCESS; return MDB_SUCCESS;
} }
static int
mdb_env_map(MDB_env *env, void *addr, int newsize)
{
MDB_page *p;
unsigned int flags = env->me_flags;
#ifdef _WIN32
int rc;
HANDLE mh;
LONG sizelo, sizehi;
sizelo = env->me_mapsize & 0xffffffff;
sizehi = env->me_mapsize >> 16 >> 16; /* only needed on Win64 */
/* Windows won't create mappings for zero length files.
* Just allocate the maxsize right now.
*/
if (newsize) {
if (SetFilePointer(env->me_fd, sizelo, &sizehi, 0) != (DWORD)sizelo
|| !SetEndOfFile(env->me_fd)
|| SetFilePointer(env->me_fd, 0, NULL, 0) != 0)
return ErrCode();
}
mh = CreateFileMapping(env->me_fd, NULL, flags & MDB_WRITEMAP ?
PAGE_READWRITE : PAGE_READONLY,
sizehi, sizelo, NULL);
if (!mh)
return ErrCode();
env->me_map = MapViewOfFileEx(mh, flags & MDB_WRITEMAP ?
FILE_MAP_WRITE : FILE_MAP_READ,
0, 0, env->me_mapsize, addr);
rc = env->me_map ? 0 : ErrCode();
CloseHandle(mh);
if (rc)
return rc;
#else
int prot = PROT_READ;
if (flags & MDB_WRITEMAP) {
prot |= PROT_WRITE;
if (newsize && ftruncate(env->me_fd, env->me_mapsize) < 0)
return ErrCode();
}
env->me_map = mmap(addr, env->me_mapsize, prot, MAP_SHARED,
env->me_fd, 0);
if (env->me_map == MAP_FAILED) {
env->me_map = NULL;
return ErrCode();
}
/* Turn off readahead. It's harmful when the DB is larger than RAM. */
#ifdef MADV_RANDOM
madvise(env->me_map, env->me_mapsize, MADV_RANDOM);
#else
#ifdef POSIX_MADV_RANDOM
posix_madvise(env->me_map, env->me_mapsize, POSIX_MADV_RANDOM);
#endif /* POSIX_MADV_RANDOM */
#endif /* MADV_RANDOM */
#endif /* _WIN32 */
/* Can happen because the address argument to mmap() is just a
* hint. mmap() can pick another, e.g. if the range is in use.
* The MAP_FIXED flag would prevent that, but then mmap could
* instead unmap existing pages to make room for the new map.
*/
if (addr && env->me_map != addr)
return EBUSY; /* TODO: Make a new MDB_* error code? */
p = (MDB_page *)env->me_map;
env->me_metas[0] = METADATA(p);
env->me_metas[1] = (MDB_meta *)((char *)env->me_metas[0] + env->me_psize);
return MDB_SUCCESS;
}
int int
mdb_env_set_mapsize(MDB_env *env, size_t size) mdb_env_set_mapsize(MDB_env *env, size_t size)
{ {
if (env->me_map) /* If env is already open, caller is responsible for making
return EINVAL; * sure there are no active txns.
*/
if (env->me_map) {
int rc;
void *old;
if (env->me_txn)
return EINVAL;
if (!size)
size = env->me_metas[mdb_env_pick_meta(env)]->mm_mapsize;
munmap(env->me_map, env->me_mapsize);
env->me_mapsize = size;
old = (env->me_flags & MDB_FIXEDMAP) ? env->me_map : NULL;
rc = mdb_env_map(env, old, 1);
if (rc)
return rc;
}
env->me_mapsize = size; env->me_mapsize = size;
if (env->me_psize) if (env->me_psize)
env->me_maxpg = env->me_mapsize / env->me_psize; env->me_maxpg = env->me_mapsize / env->me_psize;
@ -3282,12 +3368,17 @@ static int
mdb_env_open2(MDB_env *env) mdb_env_open2(MDB_env *env)
{ {
unsigned int flags = env->me_flags; unsigned int flags = env->me_flags;
int i, newenv = 0; int i, newenv = 0, rc;
MDB_meta meta; MDB_meta meta;
MDB_page *p;
#ifndef _WIN32 #ifdef _WIN32
int prot; /* See if we should use QueryLimited */
#endif rc = GetVersion();
if ((rc & 0xff) > 5)
env->me_pidquery = MDB_PROCESS_QUERY_LIMITED_INFORMATION;
else
env->me_pidquery = PROCESS_QUERY_INFORMATION;
#endif /* _WIN32 */
memset(&meta, 0, sizeof(meta)); memset(&meta, 0, sizeof(meta));
@ -3296,6 +3387,9 @@ mdb_env_open2(MDB_env *env)
return i; return i;
DPUTS("new mdbenv"); DPUTS("new mdbenv");
newenv = 1; newenv = 1;
GET_PAGESIZE(env->me_psize);
} else {
env->me_psize = meta.mm_psize;
} }
/* Was a mapsize configured? */ /* Was a mapsize configured? */
@ -3313,66 +3407,9 @@ mdb_env_open2(MDB_env *env)
env->me_mapsize = minsize; env->me_mapsize = minsize;
} }
#ifdef _WIN32 rc = mdb_env_map(env, meta.mm_address, newenv);
{ if (rc)
int rc; return rc;
HANDLE mh;
LONG sizelo, sizehi;
sizelo = env->me_mapsize & 0xffffffff;
sizehi = env->me_mapsize >> 16 >> 16; /* only needed on Win64 */
/* See if we should use QueryLimited */
rc = GetVersion();
if ((rc & 0xff) > 5)
env->me_pidquery = MDB_PROCESS_QUERY_LIMITED_INFORMATION;
else
env->me_pidquery = PROCESS_QUERY_INFORMATION;
/* Windows won't create mappings for zero length files.
* Just allocate the maxsize right now.
*/
if (newenv) {
if (SetFilePointer(env->me_fd, sizelo, &sizehi, 0) != (DWORD)sizelo
|| !SetEndOfFile(env->me_fd)
|| SetFilePointer(env->me_fd, 0, NULL, 0) != 0)
return ErrCode();
}
mh = CreateFileMapping(env->me_fd, NULL, flags & MDB_WRITEMAP ?
PAGE_READWRITE : PAGE_READONLY,
sizehi, sizelo, NULL);
if (!mh)
return ErrCode();
env->me_map = MapViewOfFileEx(mh, flags & MDB_WRITEMAP ?
FILE_MAP_WRITE : FILE_MAP_READ,
0, 0, env->me_mapsize, meta.mm_address);
rc = env->me_map ? 0 : ErrCode();
CloseHandle(mh);
if (rc)
return rc;
}
#else
i = MAP_SHARED;
prot = PROT_READ;
if (flags & MDB_WRITEMAP) {
prot |= PROT_WRITE;
if (ftruncate(env->me_fd, env->me_mapsize) < 0)
return ErrCode();
}
env->me_map = mmap(meta.mm_address, env->me_mapsize, prot, i,
env->me_fd, 0);
if (env->me_map == MAP_FAILED) {
env->me_map = NULL;
return ErrCode();
}
/* Turn off readahead. It's harmful when the DB is larger than RAM. */
#ifdef MADV_RANDOM
madvise(env->me_map, env->me_mapsize, MADV_RANDOM);
#else
#ifdef POSIX_MADV_RANDOM
posix_madvise(env->me_map, env->me_mapsize, POSIX_MADV_RANDOM);
#endif /* POSIX_MADV_RANDOM */
#endif /* MADV_RANDOM */
#endif /* _WIN32 */
if (newenv) { if (newenv) {
if (flags & MDB_FIXEDMAP) if (flags & MDB_FIXEDMAP)
@ -3381,24 +3418,11 @@ mdb_env_open2(MDB_env *env)
if (i != MDB_SUCCESS) { if (i != MDB_SUCCESS) {
return i; return i;
} }
} else if (meta.mm_address && env->me_map != meta.mm_address) {
/* Can happen because the address argument to mmap() is just a
* hint. mmap() can pick another, e.g. if the range is in use.
* The MAP_FIXED flag would prevent that, but then mmap could
* instead unmap existing pages to make room for the new map.
*/
return EBUSY; /* TODO: Make a new MDB_* error code? */
} }
env->me_psize = meta.mm_psize;
env->me_maxfree_1pg = (env->me_psize - PAGEHDRSZ) / sizeof(pgno_t) - 1; env->me_maxfree_1pg = (env->me_psize - PAGEHDRSZ) / sizeof(pgno_t) - 1;
env->me_nodemax = (env->me_psize - PAGEHDRSZ) / MDB_MINKEYS; env->me_nodemax = (env->me_psize - PAGEHDRSZ) / MDB_MINKEYS;
env->me_maxpg = env->me_mapsize / env->me_psize; env->me_maxpg = env->me_mapsize / env->me_psize;
p = (MDB_page *)env->me_map;
env->me_metas[0] = METADATA(p);
env->me_metas[1] = (MDB_meta *)((char *)env->me_metas[0] + meta.mm_psize);
#if MDB_DEBUG #if MDB_DEBUG
{ {
int toggle = mdb_env_pick_meta(env); int toggle = mdb_env_pick_meta(env);