Some tweaks to cut down on IDL stack usage. idl_intersection and idl_union

now take only two arguments instead of 3, overwriting the result onto the
first argument. (glibc2.0.7 defaults to a 2MB stack per thread; 3 IDLs at
1.5MB plus various other runtime overhead is enough to trash the stack.)
Also pass in a tmp IDL from search_candidates instead of allocating it in
each candiate function.
This commit is contained in:
Howard Chu 2001-11-26 19:32:39 +00:00
parent df28982b43
commit 763faf21b1
4 changed files with 190 additions and 246 deletions

View file

@ -19,33 +19,37 @@ static int presence_candidates(
Backend *be,
AttributeDescription *desc,
ID *ids );
static int equality_candidates(
Backend *be,
AttributeAssertion *ava,
ID *ids );
ID *ids,
ID *tmp );
static int approx_candidates(
Backend *be,
AttributeAssertion *ava,
ID *ids );
ID *ids,
ID *tmp );
static int substring_candidates(
Backend *be,
SubstringsAssertion *sub,
ID *ids );
ID *ids,
ID *tmp );
static int list_candidates(
Backend *be,
Filter *flist,
int ftype,
ID *ids );
ID *ids,
ID *tmp );
int
bdb_filter_candidates(
Backend *be,
Filter *f,
ID *ids )
ID *ids,
ID *tmp )
{
struct bdb_info *bdb = (struct bdb_info *) be->be_private;
int rc = -1;
Debug( LDAP_DEBUG_FILTER, "=> bdb_filter_candidates\n", 0, 0, 0 );
@ -67,17 +71,17 @@ bdb_filter_candidates(
case LDAP_FILTER_EQUALITY:
Debug( LDAP_DEBUG_FILTER, "\tEQUALITY\n", 0, 0, 0 );
rc = equality_candidates( be, f->f_ava, ids );
rc = equality_candidates( be, f->f_ava, ids, tmp );
break;
case LDAP_FILTER_APPROX:
Debug( LDAP_DEBUG_FILTER, "\tAPPROX\n", 0, 0, 0 );
rc = approx_candidates( be, f->f_ava, ids );
rc = approx_candidates( be, f->f_ava, ids, tmp );
break;
case LDAP_FILTER_SUBSTRINGS:
Debug( LDAP_DEBUG_FILTER, "\tSUBSTRINGS\n", 0, 0, 0 );
rc = substring_candidates( be, f->f_sub, ids );
rc = substring_candidates( be, f->f_sub, ids, tmp );
break;
case LDAP_FILTER_GE:
@ -95,26 +99,23 @@ bdb_filter_candidates(
case LDAP_FILTER_NOT:
/* no indexing to support NOT filters */
Debug( LDAP_DEBUG_FILTER, "\tNOT\n", 0, 0, 0 );
BDB_IDL_ALL( bdb, ids );
rc = 0;
break;
case LDAP_FILTER_AND:
Debug( LDAP_DEBUG_FILTER, "\tAND\n", 0, 0, 0 );
rc = list_candidates( be,
f->f_and, LDAP_FILTER_AND, ids );
f->f_and, LDAP_FILTER_AND, ids, tmp );
break;
case LDAP_FILTER_OR:
Debug( LDAP_DEBUG_FILTER, "\tOR\n", 0, 0, 0 );
rc = list_candidates( be,
f->f_or, LDAP_FILTER_OR, ids );
f->f_or, LDAP_FILTER_OR, ids, tmp );
break;
default:
Debug( LDAP_DEBUG_FILTER, "\tUNKNOWN %d\n",
f->f_choice, 0, 0 );
BDB_IDL_ALL( bdb, ids );
}
Debug( LDAP_DEBUG_FILTER,
@ -131,62 +132,50 @@ list_candidates(
Backend *be,
Filter *flist,
int ftype,
ID *ids )
ID *ids,
ID *tmp )
{
struct bdb_info *bdb = (struct bdb_info *) be->be_private;
int rc = 0;
Filter *f;
ID tmp[BDB_IDL_UM_SIZE];
/* Systems that can't increase thread stack size will die with these
* structures allocated on the stack. */
#if !defined(LDAP_PVT_THREAD_STACK_SIZE) || (LDAP_PVT_THREAD_STACK_SIZE == 0)
ID *save = ch_malloc(BDB_IDL_UM_SIZEOF);
#else
ID save[BDB_IDL_UM_SIZE];
ID *i1, *i2, *i3, *t;
#endif
Debug( LDAP_DEBUG_FILTER, "=> bdb_list_candidates 0x%x\n", ftype, 0, 0 );
/* Swap i1/i2/i3 pointers around to avoid a bunch of BDB_IDL_CPYs
* inside the loop
*/
i1 = ids;
i2 = save;
i3 = tmp;
BDB_IDL_ZERO( tmp );
if ( ftype == LDAP_FILTER_OR ) {
BDB_IDL_ALL( bdb, save );
BDB_IDL_ZERO( ids );
} else {
BDB_IDL_CPY( save, ids );
}
for ( f = flist; f != NULL; f = f->f_next ) {
BDB_IDL_ZERO( i2 );
rc = bdb_filter_candidates( be, f, i2 );
rc = bdb_filter_candidates( be, f, save, tmp );
if ( rc != 0 ) {
/* Error: treat as undefined */
continue;
}
if ( f == flist ) {
/* We're just starting out... */
t = i3;
i3 = i2;
i2 = t;
continue;
}
t = i1;
i1 = i3;
i3 = t;
if ( ftype == LDAP_FILTER_AND ) {
bdb_idl_intersection( i1, i2, i3 );
if( BDB_IDL_IS_ZERO( i3 ) ) {
if ( i3 != ids ) {
BDB_IDL_ZERO( ids );
i3 = ids;
}
bdb_idl_intersection( ids, save );
if( BDB_IDL_IS_ZERO( ids ) )
break;
}
} else {
bdb_idl_union( i1, i2, i3 );
bdb_idl_union( ids, save );
BDB_IDL_ALL( bdb, save );
}
}
if (i3 != ids)
BDB_IDL_CPY(ids, i3);
#if !defined(LDAP_PVT_THREAD_STACK_SIZE) || (LDAP_PVT_THREAD_STACK_SIZE == 0)
free(save);
#endif
Debug( LDAP_DEBUG_FILTER,
"<= bdb_list_candidates: id=%ld first=%ld last=%ld\n",
@ -202,14 +191,12 @@ presence_candidates(
AttributeDescription *desc,
ID *ids )
{
struct bdb_info *bdb = (struct bdb_info *) be->be_private;
DB *db;
int rc;
slap_mask_t mask;
struct berval prefix = {0};
Debug( LDAP_DEBUG_TRACE, "=> bdb_presence_candidates\n", 0, 0, 0 );
BDB_IDL_ALL( bdb, ids );
rc = bdb_index_param( be, desc, LDAP_FILTER_PRESENT,
&db, &mask, &prefix );
@ -262,9 +249,9 @@ static int
equality_candidates(
Backend *be,
AttributeAssertion *ava,
ID *ids )
ID *ids,
ID *tmp )
{
struct bdb_info *bdb = (struct bdb_info *) be->be_private;
DB *db;
int i;
int rc;
@ -272,12 +259,8 @@ equality_candidates(
struct berval prefix = {0};
struct berval **keys = NULL;
MatchingRule *mr;
ID tmp[BDB_IDL_UM_SIZE];
ID save[BDB_IDL_UM_SIZE];
ID *i1, *i2, *i3, *t;
Debug( LDAP_DEBUG_TRACE, "=> bdb_equality_candidates\n", 0, 0, 0 );
BDB_IDL_ALL( bdb, ids );
rc = bdb_index_param( be, ava->aa_desc, LDAP_FILTER_EQUALITY,
&db, &mask, &prefix );
@ -327,17 +310,8 @@ equality_candidates(
return 0;
}
/* Swap i1/i2/i3 pointers around to avoid a bunch of BDB_IDL_CPYs
* inside the loop
*/
i1 = ids;
i2 = save;
i3 = tmp;
BDB_IDL_ALL( bdb, tmp );
for ( i= 0; keys[i] != NULL; i++ ) {
rc = bdb_key_read( be, db, NULL, keys[i], i2 );
rc = bdb_key_read( be, db, NULL, keys[i], tmp );
if( rc != LDAP_SUCCESS ) {
Debug( LDAP_DEBUG_TRACE,
@ -346,49 +320,19 @@ equality_candidates(
break;
}
if( BDB_IDL_IS_ZERO( i2 ) ) {
if( BDB_IDL_IS_ZERO( tmp ) ) {
Debug( LDAP_DEBUG_TRACE,
"<= bdb_equality_candidates NULL\n",
0, 0, 0 );
if (i3 != ids)
BDB_IDL_ZERO( ids );
i3 = ids;
BDB_IDL_ZERO( ids );
break;
}
/* We've only gotten one set of IDs, nothing to intersect
* with yet. Just go back and get another set of IDs.
*/
if (i == 0)
{
t = i3;
i3 = i2;
i2 = t;
continue;
}
bdb_idl_intersection( ids, tmp );
/* Swap ids and save every time we get a new intersection.
* This avoids multiple copies... The result is always
* pointed to by i3.
*/
t = i1;
i1 = i3;
i3 = t;
bdb_idl_intersection( i1, i2, i3 );
if( BDB_IDL_IS_ZERO( i3 ) ) {
if ( i3 != ids ) {
BDB_IDL_ZERO( ids );
i3 = ids;
}
if( BDB_IDL_IS_ZERO( ids ) )
break;
}
}
/* If we didn't end up with the result in ids, copy it now. */
if (i3 != ids)
BDB_IDL_CPY(ids, i3);
ber_bvecfree( keys );
@ -405,9 +349,9 @@ static int
approx_candidates(
Backend *be,
AttributeAssertion *ava,
ID *ids )
ID *ids,
ID *tmp )
{
struct bdb_info *bdb = (struct bdb_info *) be->be_private;
DB *db;
int i;
int rc;
@ -415,12 +359,8 @@ approx_candidates(
struct berval prefix = {0};
struct berval **keys = NULL;
MatchingRule *mr;
ID tmp[BDB_IDL_UM_SIZE];
ID save[BDB_IDL_UM_SIZE];
ID *i1, *i2, *i3, *t;
Debug( LDAP_DEBUG_TRACE, "=> bdb_approx_candidates\n", 0, 0, 0 );
BDB_IDL_ALL( bdb, ids );
rc = bdb_index_param( be, ava->aa_desc, LDAP_FILTER_APPROX,
&db, &mask, &prefix );
@ -475,17 +415,8 @@ approx_candidates(
return 0;
}
/* Swap i1/i2/i3 pointers around to avoid a bunch of BDB_IDL_CPYs
* inside the loop
*/
i1 = ids;
i2 = save;
i3 = tmp;
BDB_IDL_ALL( bdb, tmp );
for ( i= 0; keys[i] != NULL; i++ ) {
rc = bdb_key_read( be, db, NULL, keys[i], i2 );
rc = bdb_key_read( be, db, NULL, keys[i], tmp );
if( rc != LDAP_SUCCESS ) {
Debug( LDAP_DEBUG_TRACE, "<= bdb_approx_candidates key read failed (%d)\n",
@ -493,39 +424,18 @@ approx_candidates(
break;
}
if( BDB_IDL_IS_ZERO( i2 ) ) {
if( BDB_IDL_IS_ZERO( tmp ) ) {
Debug( LDAP_DEBUG_TRACE, "<= bdb_approx_candidates NULL\n",
0, 0, 0 );
if (i3 != ids)
BDB_IDL_ZERO( ids );
i3 = ids;
BDB_IDL_ZERO( ids );
break;
}
if (i == 0)
{
t = i3;
i3 = i2;
i2 = t;
continue;
}
bdb_idl_intersection( ids, tmp );
t = i1;
i1 = i3;
i3 = t;
bdb_idl_intersection( i1, i2, i3 );
if( BDB_IDL_IS_ZERO( i3 ) ) {
if ( i3 != ids ) {
BDB_IDL_ZERO( ids );
i3 = ids;
}
if( BDB_IDL_IS_ZERO( ids ) )
break;
}
}
if (i3 != ids)
BDB_IDL_CPY(ids, i3);
ber_bvecfree( keys );
@ -541,9 +451,9 @@ static int
substring_candidates(
Backend *be,
SubstringsAssertion *sub,
ID *ids )
ID *ids,
ID *tmp )
{
struct bdb_info *bdb = (struct bdb_info *) be->be_private;
DB *db;
int i;
int rc;
@ -551,12 +461,8 @@ substring_candidates(
struct berval prefix = {0};
struct berval **keys = NULL;
MatchingRule *mr;
ID tmp[BDB_IDL_UM_SIZE];
ID save[BDB_IDL_UM_SIZE];
ID *i1, *i2, *i3, *t;
Debug( LDAP_DEBUG_TRACE, "=> bdb_substring_candidates\n", 0, 0, 0 );
BDB_IDL_ALL( bdb, ids );
rc = bdb_index_param( be, sub->sa_desc, LDAP_FILTER_SUBSTRINGS,
&db, &mask, &prefix );
@ -608,17 +514,8 @@ substring_candidates(
return 0;
}
/* Swap i1/i2/i3 pointers around to avoid a bunch of BDB_IDL_CPYs
* inside the loop
*/
i1 = ids;
i2 = save;
i3 = tmp;
BDB_IDL_ALL( bdb, tmp );
for ( i= 0; keys[i] != NULL; i++ ) {
rc = bdb_key_read( be, db, NULL, keys[i], i2 );
rc = bdb_key_read( be, db, NULL, keys[i], tmp );
if( rc != LDAP_SUCCESS ) {
Debug( LDAP_DEBUG_TRACE, "<= bdb_substring_candidates key read failed (%d)\n",
@ -626,39 +523,18 @@ substring_candidates(
break;
}
if( BDB_IDL_IS_ZERO( i2 ) ) {
if( BDB_IDL_IS_ZERO( tmp ) ) {
Debug( LDAP_DEBUG_TRACE, "<= bdb_substring_candidates NULL\n",
0, 0, 0 );
if (i3 != ids)
BDB_IDL_ZERO( ids );
i3 = ids;
BDB_IDL_ZERO( ids );
break;
}
if (i == 0)
{
t = i3;
i3 = i2;
i2 = t;
continue;
}
bdb_idl_intersection( ids, tmp );
t = i1;
i1 = i3;
i3 = t;
bdb_idl_intersection( i1, i2, i3 );
if( BDB_IDL_IS_ZERO( i3 ) ) {
if ( i3 != ids ) {
BDB_IDL_ZERO( ids );
i3 = ids;
}
if( BDB_IDL_IS_ZERO( ids ) )
break;
}
}
if (i3 != ids)
BDB_IDL_CPY(ids, i3);
ber_bvecfree( keys );

View file

@ -437,126 +437,153 @@ bdb_idl_delete_key(
/*
* idl_intersection - return a intersection b
* idl_intersection - return a = a intersection b
*/
int
bdb_idl_intersection(
ID *a,
ID *b,
ID *ids )
ID *b )
{
ID ida, idb;
ID cursora = 0, cursorb = 0;
ID idmax, idmin;
ID cursora = 0, cursorb = 0, cursorc;
int swap = 0;
if ( BDB_IDL_IS_ZERO( a ) || BDB_IDL_IS_ZERO( b ) ) {
ids[0] = 0;
a[0] = 0;
return 0;
}
idmin = IDL_MAX( BDB_IDL_FIRST(a), BDB_IDL_FIRST(b) );
idmax = IDL_MIN( BDB_IDL_LAST(a), BDB_IDL_LAST(b) );
if ( idmin > idmax ) {
a[0] = 0;
return 0;
} else if ( idmin == idmax ) {
a[0] = 1;
a[1] = idmin;
return 0;
}
if ( BDB_IDL_IS_RANGE( a ) && BDB_IDL_IS_RANGE(b) ) {
ids[0] = NOID;
ids[1] = IDL_MAX( BDB_IDL_FIRST(a), BDB_IDL_FIRST(b) );
ids[2] = IDL_MIN( BDB_IDL_LAST(a), BDB_IDL_FIRST(b) );
if ( ids[1] == ids[2] ) {
ids[0] = 1;
} else if( ids[1] > ids[2] ) {
ids[0] = 0;
}
a[1] = idmin;
a[2] = idmax;
return 0;
}
if( BDB_IDL_IS_RANGE( a ) ) {
if ( BDB_IDL_IS_RANGE( a ) ) {
ID *tmp = a;
a = b;
b = tmp;
swap = 1;
}
ida = bdb_idl_first( a, &cursora ),
if ( BDB_IDL_IS_RANGE( b ) && BDB_IDL_FIRST( b ) <= idmin &&
BDB_IDL_LAST( b ) >= idmax) {
if (idmax - idmin + 1 == a[0])
{
a[0] = NOID;
a[1] = idmin;
a[2] = idmax;
}
goto done;
}
ida = bdb_idl_first( a, &cursora );
idb = bdb_idl_first( b, &cursorb );
cursorc = 0;
ids[0] = 0;
while( ida < idmin )
ida = bdb_idl_next( a, &cursora );
while( idb < idmin )
idb = bdb_idl_next( b, &cursorb );
while( ida != NOID || idb != NOID ) {
while( ida <= idmax || idb <= idmax ) {
if( ida == idb ) {
ids[++ids[0]] = ida;
a[++cursorc] = ida;
ida = bdb_idl_next( a, &cursora );
idb = bdb_idl_next( b, &cursorb );
if( BDB_IDL_IS_RANGE( b ) && idb < ida ) {
if( ida > BDB_IDL_LAST( b ) ) {
idb = NOID;
} else {
idb = ida;
cursorb = ida;
}
}
} else if ( ida < idb ) {
ida = bdb_idl_next( a, &cursora );
} else {
idb = bdb_idl_next( b, &cursorb );
}
}
a[0] = cursorc;
done:
if (swap)
BDB_IDL_CPY( b, a );
return 0;
}
/*
* idl_union - return a union b
* idl_union - return a = a union b
*/
int
bdb_idl_union(
ID *a,
ID *b,
ID *ids )
ID *b )
{
ID ida, idb;
ID cursora = 0, cursorb = 0;
ID cursora = 0, cursorb = 0, cursorc;
if ( BDB_IDL_IS_ZERO( a ) ) {
BDB_IDL_CPY( ids, b );
if ( BDB_IDL_IS_ZERO( b ) ) {
return 0;
}
if ( BDB_IDL_IS_ZERO( b ) ) {
BDB_IDL_CPY( ids, a );
if ( BDB_IDL_IS_ZERO( a ) ) {
BDB_IDL_CPY( a, b );
return 0;
}
if ( BDB_IDL_IS_RANGE( a ) || BDB_IDL_IS_RANGE(b) ) {
ids[0] = NOID;
ids[1] = IDL_MIN( BDB_IDL_FIRST(a), BDB_IDL_FIRST(b) );
ids[2] = IDL_MAX( BDB_IDL_LAST(a), BDB_IDL_FIRST(b) );
over: a[1] = IDL_MIN( BDB_IDL_FIRST(a), BDB_IDL_FIRST(b) );
a[2] = IDL_MAX( BDB_IDL_LAST(a), BDB_IDL_LAST(b) );
return 0;
}
ida = bdb_idl_first( a, &cursora );
idb = bdb_idl_first( b, &cursorb );
ids[0] = 0;
cursorc = b[0];
/* The distinct elements of a are cat'd to b */
while( ida != NOID || idb != NOID ) {
if( ++ids[0] > BDB_IDL_UM_MAX ) {
ids[0] = NOID;
ids[2] = IDL_MAX( BDB_IDL_LAST(a), BDB_IDL_LAST(b) );
break;
}
if ( ida < idb ) {
ids[ids[0]] = ida;
if( ++cursorc > BDB_IDL_UM_MAX ) {
a[0] = NOID;
goto over;
}
b[cursorc] = ida;
ida = bdb_idl_next( a, &cursora );
} else if ( ida > idb ) {
ids[ids[0]] = idb;
idb = bdb_idl_next( b, &cursorb );
} else {
ids[ids[0]] = ida;
ida = bdb_idl_next( a, &cursora );
if ( ida == idb )
ida = bdb_idl_next( a, &cursora );
idb = bdb_idl_next( b, &cursorb );
}
}
/* b is copied back to a in sorted order */
a[0] = cursorc;
cursora = 1;
cursorb = 1;
cursorc = b[0]+1;
while (cursorb <= b[0] || cursorc <= a[0]) {
if (cursorc > a[0])
idb = NOID;
else
idb = b[cursorc];
if (b[cursorb] < idb)
a[cursora++] = b[cursorb++];
else {
a[cursora++] = idb;
cursorc++;
}
}
return 0;
}

View file

@ -119,7 +119,8 @@ void bdb_errcall( const char *pfx, char * msg );
int bdb_filter_candidates(
Backend *be,
Filter *f,
ID *ids );
ID *ids,
ID *tmp );
/*
* group.c
@ -185,23 +186,23 @@ int bdb_idl_delete_key(
DBT *key,
ID id );
#if 0
int
bdb_idl_notin(
ID *a,
ID *b,
ID *ids );
#endif
int
bdb_idl_intersection(
ID *a,
ID *b,
ID *ids );
ID *b );
int
bdb_idl_union(
ID *a,
ID *b,
ID *ids );
ID *b );
ID bdb_idl_first( ID *ids, ID *cursor );
ID bdb_idl_next( ID *ids, ID *cursor );

View file

@ -231,6 +231,7 @@ bdb_search(
rc = base_candidate( be, e, candidates );
} else {
BDB_IDL_ALL( bdb, candidates );
rc = search_candidates( be, e, filter,
scope, deref, manageDSAit, candidates );
}
@ -468,6 +469,39 @@ static int base_candidate(
return 0;
}
/* Is "objectClass=xx" mentioned anywhere in this filter? Presence
* doesn't count, we're looking for explicit values.
*/
static int oc_filter(
Filter *f
)
{
int rc = 0;
switch(f->f_choice) {
case LDAP_FILTER_EQUALITY:
case LDAP_FILTER_APPROX:
if (f->f_av_desc == slap_schema.si_ad_objectClass)
rc = 1;
break;
case LDAP_FILTER_SUBSTRINGS:
if (f->f_sub_desc == slap_schema.si_ad_objectClass)
rc = 1;
break;
case LDAP_FILTER_AND:
case LDAP_FILTER_OR:
for (f=f->f_and; f; f=f->f_next)
if ((rc = oc_filter(f)))
break;
break;
default:
break;
}
return rc;
}
static int search_candidates(
BackendDB *be,
Entry *e,
@ -479,6 +513,7 @@ static int search_candidates(
{
int rc;
Filter f, fand, rf, xf;
ID tmp[BDB_IDL_UM_SIZE];
AttributeAssertion aa_ref;
struct bdb_info *bdb = (struct bdb_info *) be->be_private;
#ifdef BDB_ALIASES
@ -494,7 +529,11 @@ static int search_candidates(
xf.f_choice = LDAP_FILTER_OR;
xf.f_next = NULL;
if( !manageDSAit ) {
/* If the user's filter doesn't mention objectClass, or if
* it just uses objectClass=*, these clauses are redundant.
*/
if (oc_filter(filter)) {
if( !manageDSAit ) {
/* match referrals */
static struct berval bv_ref = { sizeof("REFERRAL")-1, "REFERRAL" };
rf.f_choice = LDAP_FILTER_EQUALITY;
@ -503,10 +542,10 @@ static int search_candidates(
rf.f_av_value = &bv_ref;
rf.f_next = xf.f_or;
xf.f_or = &rf;
}
}
#ifdef BDB_ALIASES
if( deref & LDAP_DEREF_SEARCHING ) {
if( deref & LDAP_DEREF_SEARCHING ) {
/* match aliases */
static struct berval bv_alias = { sizeof("ALIAS")-1, "ALIAS" };
af.f_choice = LDAP_FILTER_EQUALITY;
@ -515,8 +554,9 @@ static int search_candidates(
af.f_av_value = &bv_alias;
af.f_next = xf.f_or;
xf.f_or = &af;
}
}
#endif
}
f.f_next = NULL;
f.f_choice = LDAP_FILTER_AND;
@ -529,7 +569,7 @@ static int search_candidates(
#ifdef BDB_FILTER_INDICES
rc = bdb_filter_candidates( be, &f, ids );
rc = bdb_filter_candidates( be, &f, ids, tmp );
#else
/* FIXME: Original code:
BDB_IDL_ID( bdb, ids, e->e_id );