diff --git a/doc/man/man5/slapd-mdb.5 b/doc/man/man5/slapd-mdb.5 index ccfc7e91a0..e609e4e9a8 100644 --- a/doc/man/man5/slapd-mdb.5 +++ b/doc/man/man5/slapd-mdb.5 @@ -20,6 +20,18 @@ supports subtree renames. It is both more space-efficient and more execution-efficient than the \fBbdb\fP backend, while being overall much simpler to manage. .SH CONFIGURATION +These +.B slapd.conf +options apply to the \fBmdb\fP backend. +That is, they must follow a "backend mdb" line and +come before any subsequent "backend" or "database" lines. +.TP +.BI idlexp \ +Specify a power of 2 for the maximum size of an index slot. +The default is 16, yielding a maximum slot size of 2^16 or 65536. +Once set, this option applies to every \fBmdb\fP database instance. +.LP + These .B slapd.conf options apply to the \fBmdb\fP backend database. diff --git a/servers/slapd/back-mdb/config.c b/servers/slapd/back-mdb/config.c index 42e8f0d915..19d1942477 100644 --- a/servers/slapd/back-mdb/config.c +++ b/servers/slapd/back-mdb/config.c @@ -22,13 +22,16 @@ #include #include "back-mdb.h" +#include "idl.h" #include "config.h" #include "lutil.h" #include "ldap_rq.h" + static ConfigDriver mdb_cf_gen; +static ConfigDriver mdb_bk_cfg; enum { MDB_CHKPT = 1, @@ -41,9 +44,15 @@ enum { MDB_MODE, MDB_SSTACK, MDB_MULTIVAL, + MDB_IDLEXP, }; static ConfigTable mdbcfg[] = { + { "idlexp", "log", 2, 2, 0, ARG_UINT|ARG_MAGIC|MDB_IDLEXP, + mdb_bk_cfg, "( OLcfgBkAt:12.1 NAME 'olcBkMdbIdlExp' " + "DESC 'Power of 2 used to set IDL size' " + "EQUALITY integerMatch " + "SYNTAX OMsInteger SINGLE-VALUE )", NULL, NULL }, { "directory", "dir", 2, 2, 0, ARG_STRING|ARG_MAGIC|MDB_DIRECTORY, mdb_cf_gen, "( OLcfgDbAt:0.1 NAME 'olcDbDirectory' " "DESC 'Directory for database content' " @@ -112,17 +121,24 @@ static ConfigTable mdbcfg[] = { }; static ConfigOCs mdbocs[] = { + { + "( OLcfgBkOc:12.1 " + "NAME 'olcMdbBkConfig' " + "DESC 'MDB backend configuration' " + "SUP olcBackendConfig " + "MAY olcBkMdbIdlExp )", + Cft_Backend, mdbcfg }, { "( OLcfgDbOc:12.1 " "NAME 'olcMdbConfig' " - "DESC 'MDB backend configuration' " + "DESC 'MDB database configuration' " "SUP olcDatabaseConfig " "MUST olcDbDirectory " "MAY ( olcDbCheckpoint $ olcDbEnvFlags $ " "olcDbNoSync $ olcDbIndex $ olcDbMaxReaders $ olcDbMaxSize $ " "olcDbMode $ olcDbSearchStack $ olcDbMaxEntrySize $ olcDbRtxnSize $ " "olcDbMultival ) )", - Cft_Database, mdbcfg }, + Cft_Database, mdbcfg+1 }, { NULL, 0, NULL } }; @@ -135,6 +151,29 @@ static slap_verbmasks mdb_envflags[] = { { BER_BVNULL, 0 } }; +static int +mdb_bk_cfg( ConfigArgs *c ) +{ + int rc = 0; + if ( c->op == SLAP_CONFIG_EMIT ) { + if ( MDB_idl_logn != MDB_IDL_LOGN ) + c->value_int = MDB_idl_logn; + else + rc = 1; + } else if ( c->op == LDAP_MOD_DELETE ) { + MDB_idl_logn = 0; + mdb_idl_reset(); + } else { + if ( c->value_int >= MDB_IDL_LOGN && c->value_int < sizeof(int) * CHAR_BIT ) { + MDB_idl_logn = c->value_int; + mdb_idl_reset(); + } else { + rc = 1; + } + } + return rc; +} + /* perform periodic syncs */ static void * mdb_checkpoint( void *ctx, void *arg ) diff --git a/servers/slapd/back-mdb/dn2id.c b/servers/slapd/back-mdb/dn2id.c index 1ac804f227..7020bc6ab2 100644 --- a/servers/slapd/back-mdb/dn2id.c +++ b/servers/slapd/back-mdb/dn2id.c @@ -664,7 +664,7 @@ mdb_idscope( ptr += data.mv_size - sizeof(ID); memcpy( &id, ptr, sizeof(ID) ); if ( id == base ) { - if ( res[0] >= MDB_IDL_DB_SIZE-1 ) { + if ( res[0] >= MDB_idl_db_max ) { /* too many aliases in scope. Fallback to range */ MDB_IDL_RANGE( res, MDB_IDL_FIRST( ids ), MDB_IDL_LAST( ids )); goto leave; diff --git a/servers/slapd/back-mdb/filterindex.c b/servers/slapd/back-mdb/filterindex.c index e3895d2767..0183ab6955 100644 --- a/servers/slapd/back-mdb/filterindex.c +++ b/servers/slapd/back-mdb/filterindex.c @@ -270,7 +270,7 @@ comp_list_candidates( continue; } MDB_IDL_ZERO( save ); - rc = comp_candidates( op, rtxn, mra, f, save, tmp, save+MDB_IDL_UM_SIZE ); + rc = comp_candidates( op, rtxn, mra, f, save, tmp, save+MDB_idl_um_size ); if ( rc != 0 ) { if ( ftype == LDAP_COMP_FILTER_AND ) { @@ -580,7 +580,7 @@ list_candidates( } MDB_IDL_ZERO( save ); rc = mdb_filter_candidates( op, rtxn, f, save, tmp, - save+MDB_IDL_UM_SIZE ); + save+MDB_idl_um_size ); if ( rc != 0 ) { if ( ftype == LDAP_FILTER_AND ) { diff --git a/servers/slapd/back-mdb/idl.c b/servers/slapd/back-mdb/idl.c index c605b30dd0..637eaa5b9f 100644 --- a/servers/slapd/back-mdb/idl.c +++ b/servers/slapd/back-mdb/idl.c @@ -22,6 +22,12 @@ #include "back-mdb.h" #include "idl.h" +unsigned int MDB_idl_logn = MDB_IDL_LOGN; +unsigned int MDB_idl_db_size = 1 << MDB_IDL_LOGN; +unsigned int MDB_idl_um_size = 1 << (MDB_IDL_LOGN+1); +unsigned int MDB_idl_db_max = (1 << MDB_IDL_LOGN) - 1; +unsigned int MDB_idl_um_max = (1 << (MDB_IDL_LOGN+1) - 1); + #define IDL_MAX(x,y) ( (x) > (y) ? (x) : (y) ) #define IDL_MIN(x,y) ( (x) < (y) ? (x) : (y) ) #define IDL_CMP(x,y) ( (x) < (y) ? -1 : (x) > (y) ) @@ -67,6 +73,17 @@ static void idl_dump( ID *ids ) #endif /* IDL_DEBUG > 1 */ #endif /* IDL_DEBUG > 0 */ +void mdb_idl_reset() +{ + if ( !MDB_idl_logn ) + MDB_idl_logn = MDB_IDL_LOGN; + + MDB_idl_db_size = 1 << MDB_idl_logn; + MDB_idl_um_size = 1 << (MDB_idl_logn+1); + MDB_idl_db_max = MDB_idl_db_size - 1; + MDB_idl_um_max = MDB_idl_um_size - 1; +} + unsigned mdb_idl_search( ID *ids, ID id ) { #define IDL_BINARY_SEARCH 1 @@ -160,7 +177,7 @@ int mdb_idl_insert( ID *ids, ID id ) return -1; } - if ( ++ids[0] >= MDB_IDL_DB_MAX ) { + if ( ++ids[0] >= MDB_idl_db_max ) { if( id < ids[1] ) { ids[1] = id; ids[2] = ids[ids[0]-1]; @@ -447,7 +464,7 @@ mdb_idl_insert_keys( err = "c_count"; goto fail; } - if ( count >= MDB_IDL_DB_MAX ) { + if ( count >= MDB_idl_db_max ) { /* No room, convert to a range */ lo = *i; rc = mdb_cursor_get( cursor, &key, &data, MDB_LAST_DUP ); @@ -784,7 +801,7 @@ over: ida = IDL_MIN( MDB_IDL_FIRST(a), MDB_IDL_FIRST(b) ); /* The distinct elements of a are cat'd to b */ while( ida != NOID || idb != NOID ) { if ( ida < idb ) { - if( ++cursorc > MDB_IDL_UM_MAX ) { + if( ++cursorc > MDB_idl_um_max ) { goto over; } b[cursorc] = ida; @@ -949,7 +966,7 @@ int mdb_idl_append_one( ID *ids, ID id ) } } ids[0]++; - if ( ids[0] >= MDB_IDL_UM_MAX ) { + if ( ids[0] >= MDB_idl_um_max ) { ids[0] = NOID; ids[2] = id; } else { @@ -977,7 +994,7 @@ int mdb_idl_append( ID *a, ID *b ) ida = MDB_IDL_LAST( a ); idb = MDB_IDL_LAST( b ); if ( MDB_IDL_IS_RANGE( a ) || MDB_IDL_IS_RANGE(b) || - a[0] + b[0] >= MDB_IDL_UM_MAX ) { + a[0] + b[0] >= MDB_idl_um_max ) { a[2] = IDL_MAX( ida, idb ); a[1] = IDL_MIN( a[1], b[1] ); a[0] = NOID; @@ -1239,7 +1256,7 @@ int mdb_id2l_insert( ID2L ids, ID2 *id ) return -1; } - if ( ids[0].mid >= MDB_IDL_UM_MAX ) { + if ( ids[0].mid >= MDB_idl_um_max ) { /* too big */ return -2; diff --git a/servers/slapd/back-mdb/idl.h b/servers/slapd/back-mdb/idl.h index d686d0c634..1c21b7d77c 100644 --- a/servers/slapd/back-mdb/idl.h +++ b/servers/slapd/back-mdb/idl.h @@ -21,13 +21,11 @@ * limiting factors: sizeof(ID), thread stack size */ #define MDB_IDL_LOGN 16 /* DB_SIZE is 2^16, UM_SIZE is 2^17 */ -#define MDB_IDL_DB_SIZE (1<mi_multi_lo = UINT_MAX; be->be_private = mdb; - be->be_cf_ocs = be->bd_info->bi_cf_ocs; + be->be_cf_ocs = be->bd_info->bi_cf_ocs+1; #ifndef MDB_MULTIPLE_SUFFIXES SLAP_DBFLAGS( be ) |= SLAP_DBFLAG_ONE_SUFFIX; diff --git a/servers/slapd/back-mdb/search.c b/servers/slapd/back-mdb/search.c index fcda47f40d..1c1211c629 100644 --- a/servers/slapd/back-mdb/search.c +++ b/servers/slapd/back-mdb/search.c @@ -142,11 +142,11 @@ static int search_aliases( Filter af; aliases = stack; /* IDL of all aliases in the database */ - curscop = aliases + MDB_IDL_DB_SIZE; /* Aliases in the current scope */ - visited = curscop + MDB_IDL_DB_SIZE; /* IDs we've seen in this search */ - newsubs = visited + MDB_IDL_DB_SIZE; /* New subtrees we've added */ - oldsubs = newsubs + MDB_IDL_DB_SIZE; /* Subtrees added previously */ - tmp = oldsubs + MDB_IDL_DB_SIZE; /* Scratch space for deref_base() */ + curscop = aliases + MDB_idl_db_size; /* Aliases in the current scope */ + visited = curscop + MDB_idl_db_size; /* IDs we've seen in this search */ + newsubs = visited + MDB_idl_db_size; /* New subtrees we've added */ + oldsubs = newsubs + MDB_idl_db_size; /* Subtrees added previously */ + tmp = oldsubs + MDB_idl_db_size; /* Scratch space for deref_base() */ af.f_choice = LDAP_FILTER_EQUALITY; af.f_ava = &aa_alias; @@ -297,7 +297,7 @@ static ID2 *scope_chunk_get( Operation *op ) ldap_pvt_thread_pool_getkey( op->o_threadctx, (void *)scope_chunk_get, (void *)&ret, NULL ); if ( !ret ) { - ret = ch_malloc( MDB_IDL_UM_SIZE * sizeof( ID2 )); + ret = ch_malloc( MDB_idl_um_size * sizeof( ID2 )); } else { void *r2 = ret[0].mval.mv_data; ldap_pvt_thread_pool_setkey( op->o_threadctx, (void *)scope_chunk_get, @@ -416,8 +416,8 @@ mdb_search( Operation *op, SlapReply *rs ) struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private; ID id, cursor, nsubs, ncand, cscope; ID lastid = NOID; - ID candidates[MDB_IDL_UM_SIZE]; - ID iscopes[MDB_IDL_DB_SIZE]; + ID candidates[MDB_idl_um_size]; + ID iscopes[MDB_idl_db_size]; ID2 *scopes; void *stack; Entry *e = NULL, *base = NULL; @@ -1289,7 +1289,7 @@ static void *search_stack( Operation *op ) } if ( !ret ) { - ret = ch_malloc( mdb->mi_search_stack_depth * MDB_IDL_UM_SIZE + ret = ch_malloc( mdb->mi_search_stack_depth * MDB_idl_um_size * sizeof( ID ) ); if ( op->o_threadctx ) { ldap_pvt_thread_pool_setkey( op->o_threadctx, (void *)search_stack, @@ -1366,7 +1366,7 @@ static int search_candidates( /* Allocate IDL stack, plus 1 more for former tmp */ if ( depth+1 > mdb->mi_search_stack_depth ) { - stack = ch_malloc( (depth + 1) * MDB_IDL_UM_SIZE * sizeof( ID ) ); + stack = ch_malloc( (depth + 1) * MDB_idl_um_size * sizeof( ID ) ); } if( op->ors_deref & LDAP_DEREF_SEARCHING ) { @@ -1377,7 +1377,7 @@ static int search_candidates( if ( rc == LDAP_SUCCESS ) { rc = mdb_filter_candidates( op, isc->mt, f, ids, - stack, stack+MDB_IDL_UM_SIZE ); + stack, stack+MDB_idl_um_size ); } if ( depth+1 > mdb->mi_search_stack_depth ) { diff --git a/servers/slapd/back-mdb/tools.c b/servers/slapd/back-mdb/tools.c index cce8115ff3..fe4e2210e5 100644 --- a/servers/slapd/back-mdb/tools.c +++ b/servers/slapd/back-mdb/tools.c @@ -1268,14 +1268,14 @@ mdb_tool_idl_flush_one( MDB_cursor *mc, AttrIxInfo *ai, mdb_tool_idl_cache *ic ) ID id, nid; /* Freshly allocated, ignore it */ - if ( !ic->head && ic->count <= MDB_IDL_DB_SIZE ) { + if ( !ic->head && ic->count <= MDB_idl_db_size ) { return 0; } key.mv_data = ic->kstr.bv_val; key.mv_size = ic->kstr.bv_len; - if ( ic->count > MDB_IDL_DB_SIZE ) { + if ( ic->count > MDB_idl_db_size ) { while ( ic->flags & WAS_FOUND ) { rc = mdb_cursor_get( mc, &key, data, MDB_SET ); if ( rc ) { @@ -1438,7 +1438,7 @@ int mdb_tool_idl_add( ic->flags |= WAS_FOUND; nid = *(ID *)data.mv_data; if ( nid == 0 ) { - ic->count = MDB_IDL_DB_SIZE+1; + ic->count = MDB_idl_db_size+1; ic->flags |= WAS_RANGE; } else { size_t count; @@ -1451,11 +1451,11 @@ int mdb_tool_idl_add( } } /* are we a range already? */ - if ( ic->count > MDB_IDL_DB_SIZE ) { + if ( ic->count > MDB_idl_db_size ) { ic->last = id; continue; /* Are we at the limit, and converting to a range? */ - } else if ( ic->count == MDB_IDL_DB_SIZE ) { + } else if ( ic->count == MDB_idl_db_size ) { if ( ic->head ) { ic->tail->next = ax->ai_flist; ax->ai_flist = ic->head;