implement proxy quarantine (ITS#4569)

doc/man/man5/slapd-ldap.5

@@ -343,10 +343,6 @@ used by the client, otherwise the requested protocol is used.
 The proxy returns \fIunwillingToPerform\fP if an operation that is 
 incompatible with the requested protocol is attempted.
 
-.TP
-.B single\-conn {NO|yes}
-Discards current cached connection when the client rebinds.
-
 .TP
 .B proxy\-whoami {NO|yes}
 Turns on proxying of the WhoAmI extended operation. If this option is
@@ -356,12 +352,34 @@ request will be forwarded to the remote LDAP server. Other sessions will
 be handled by the local slapd, as before. This option is mainly useful
 in conjunction with Proxy Authorization.
 
+.TP
+.B quarantine <interval>,<num>[;<interval>,<num>[...]]
+Turns on quarantine of URIs that returned
+.IR LDAP_UNAVAILABLE ,
+so that an attempt to reconnect only occurs at given intervals instead
+of any time a client requests an operation.
+The pattern is: retry only after at least
+.I interval
+seconds elapsed since last attempt, for exactly
+.I num
+times; then use the next pattern.
+If
+.I num
+for the last pattern is "\fB+\fP", it retries forever; otherwise, 
+no more retries occur.
+The process can be restarted by resetting the \fIolcDbQuarantine\fP
+attribute of the database entry in the configuration backend.
+
 .TP
 .B rebind-as-user {NO|yes}
 If this option is given, the client's bind credentials are remembered
 for rebinds when chasing referrals.  Useful when
 \fBchase-referrals\fP is set to \fByes\fP, useless otherwise.
 
+.TP
+.B single\-conn {NO|yes}
+Discards current cached connection when the client rebinds.
+
 .TP
 .B t-f-support {NO|yes|discover}
 enable if the remote server supports absolute filters
@@ -400,12 +418,6 @@ as a side-effect, some of the traditional directives have been
 deprecated and should be no longer used, as they might disappear
 in future releases.
 
-.TP
-.B server <hostname[:port]>
-this directive is no longer supported.  Use the 
-.B uri
-directive as described above.
-
 .TP
 .B acl-authcDN "<administrative DN for access control purposes>"
 DN which is used to query the target server for acl checking; it
@@ -476,6 +488,12 @@ arg of
 .BR idassert-bind ,
 and will be dismissed in the future.
 
+.TP
+.B server <hostname[:port]>
+this directive is no longer supported.  Use the 
+.B uri
+directive as described above.
+
 .TP
 .B suffixmassage, map, rewrite*
 These directives are no longer supported by back-ldap; their 

doc/man/man5/slapd-meta.5

@@ -143,6 +143,24 @@ This directive, when set to
 causes the authentication to the remote servers with the pseudo-root
 identity to be deferred until actually needed by subsequent operations.
 
+.TP
+.B quarantine <interval>,<num>[;<interval>,<num>[...]]
+Turns on quarantine of URIs that returned
+.IR LDAP_UNAVAILABLE ,
+so that an attempt to reconnect only occurs at given intervals instead
+of any time a client requests an operation.
+The pattern is: retry only after at least
+.I interval
+seconds elapsed since last attempt, for exactly
+.I num
+times; then use the next pattern.
+If
+.I num
+for the last pattern is "\fB+\fP", it retries forever; otherwise, 
+no more retries occur.
+This directive must appear before any target specification;
+it affects all targets with the same pattern.
+
 .TP
 .B rebind-as-user {NO|yes}
 If this option is given, the client's bind credentials are remembered

servers/slapd/back-ldap/back-ldap.h

@@ -122,13 +122,26 @@ typedef struct ldap_avl_info_t {
 	Avlnode				*lai_tree;
 } ldap_avl_info_t;
 
+typedef struct slap_retry_info_t {
+	time_t		*ri_interval;
+	int		*ri_num;
+	int		ri_idx;
+	int		ri_count;
+	time_t		ri_last;
+
+#define SLAP_RETRYNUM_FOREVER	(-1)		/* retry forever */
+#define SLAP_RETRYNUM_TAIL	(-2)		/* end of retrynum array */
+#define SLAP_RETRYNUM_VALID(n)	((n) >= SLAP_RETRYNUM_FOREVER)	/* valid retrynum */
+#define SLAP_RETRYNUM_FINITE(n)	((n) > SLAP_RETRYNUM_FOREVER)	/* not forever */
+} slap_retry_info_t;
+
 typedef struct ldapinfo_t {
 	/* li_uri: the string that goes into ldap_initialize()
 	 * TODO: use li_acl.sb_uri instead */
-	char		*li_uri;
+	char			*li_uri;
 	/* li_bvuri: an array of each single URI that is equivalent;
 	 * to be checked for the presence of a certain item */
-	BerVarray	li_bvuri;
+	BerVarray		li_bvuri;
 	ldap_pvt_thread_mutex_t	li_uri_mutex;
 
 	LDAP_REBIND_PROC	*li_rebind_f;
@@ -231,6 +244,15 @@ typedef struct ldapinfo_t {
 
 	ldap_avl_info_t	li_conninfo;
 
+	slap_retry_info_t	li_quarantine;
+	/* NOTE: quarantine uses the connection mutex */
+	sig_atomic_t		li_isquarantined;
+#define	LDAP_BACK_FQ_NO		(0)
+#define	LDAP_BACK_FQ_YES	(1)
+#define	LDAP_BACK_FQ_RETRYING	(2)
+
+#define	LDAP_BACK_QUARANTINE(li)	( (li)->li_quarantine.ri_num != NULL )
+
 	time_t		li_network_timeout;
 	time_t		li_conn_ttl;
 	time_t		li_idle_timeout;

servers/slapd/back-ldap/bind.c

@@ -600,6 +600,37 @@ ldap_back_getconn( Operation *op, SlapReply *rs, ldap_back_send_t sendok )
 			lc_curr = { 0 };
 	int		refcnt = 1, binding = 1;
 
+	/* if the server is quarantined, and
+	 * - the current interval did not expire yet, or
+	 * - no more retries should occur,
+	 * don't return the connection */
+	if ( li->li_isquarantined ) {
+		slap_retry_info_t	*ri = &li->li_quarantine;
+		int			dont_retry = 1;
+
+		ldap_pvt_thread_mutex_lock( &li->li_conninfo.lai_mutex );
+		if ( li->li_isquarantined == LDAP_BACK_FQ_YES ) {
+			dont_retry = ( ri->ri_num[ ri->ri_idx ] == SLAP_RETRYNUM_TAIL
+				|| slap_get_time() < ri->ri_last + ri->ri_interval[ ri->ri_idx ] );
+			if ( !dont_retry ) {
+				Debug( LDAP_DEBUG_ANY,
+					"%s: ldap_back_getconn quarantine "
+					"retry block #%d try #%d.\n",
+					op->o_log_prefix, ri->ri_idx, ri->ri_count );
+				li->li_isquarantined = LDAP_BACK_FQ_RETRYING;
+			}
+		}
+		ldap_pvt_thread_mutex_unlock( &li->li_conninfo.lai_mutex );
+
+		if ( dont_retry ) {
+			rs->sr_err = LDAP_UNAVAILABLE;
+			if ( op->o_conn && ( sendok & LDAP_BACK_SENDERR ) ) {
+				send_ldap_result( op, rs );
+			}
+			return NULL;
+		}
+	}
+
 	/* Internal searches are privileged and shared. So is root. */
 	if ( op->o_do_not_cache || be_isroot( op ) ) {
 		LDAP_BACK_CONN_ISPRIV_SET( &lc_curr );
@@ -632,6 +663,7 @@ retry_lock:
 				ldap_pvt_thread_yield();
 				goto retry_lock;
 			}
+
 			refcnt = ++lc->lc_refcnt;
 			binding = ++lc->lc_binding;
 		}
@@ -742,7 +774,6 @@ retry_lock:
 		}
 
 	} else {
-		char	buf[ SLAP_TEXT_BUFLEN ];
 		int	expiring = 0;
 
 		if ( ( li->li_idle_timeout != 0 && op->o_time > lc->lc_time + li->li_idle_timeout )
@@ -760,13 +791,14 @@ retry_lock:
 		}
 
 		if ( LogTest( LDAP_DEBUG_TRACE ) ) {
+			char	buf[ SLAP_TEXT_BUFLEN ];
+
 			snprintf( buf, sizeof( buf ),
 				"conn %p fetched refcnt=%u binding=%u%s",
 				(void *)lc, refcnt, binding, expiring ? " expiring" : "" );
 			Debug( LDAP_DEBUG_TRACE,
 				"=>ldap_back_getconn: %s.\n", buf, 0, 0 );
 		}
-	
 	}
 
 #ifdef HAVE_TLS
@@ -802,6 +834,67 @@ ldap_back_release_conn_lock(
 	}
 }
 
+void
+ldap_back_quarantine(
+	Operation	*op,
+	SlapReply	*rs,
+	int		dolock )
+{
+	ldapinfo_t	*li = (ldapinfo_t *)op->o_bd->be_private;
+
+	slap_retry_info_t	*ri = &li->li_quarantine;
+
+	if ( dolock ) {
+		ldap_pvt_thread_mutex_lock( &li->li_conninfo.lai_mutex );
+	}
+
+	if ( rs->sr_err == LDAP_UNAVAILABLE ) {
+		switch ( li->li_isquarantined ) {
+		case LDAP_BACK_FQ_NO:
+			Debug( LDAP_DEBUG_ANY,
+				"%s: ldap_back_quarantine enter.\n",
+				op->o_log_prefix, 0, 0 );
+
+			ri->ri_idx = 0;
+			ri->ri_count = 0;
+			break;
+
+		case LDAP_BACK_FQ_RETRYING:
+			Debug( LDAP_DEBUG_ANY,
+				"%s: ldap_back_quarantine block #%d try #%d failed.\n",
+				op->o_log_prefix, ri->ri_idx, ri->ri_count );
+
+			++ri->ri_count;
+			if ( ri->ri_num[ ri->ri_idx ] != SLAP_RETRYNUM_FOREVER
+				&& ri->ri_count == ri->ri_num[ ri->ri_idx ] )
+			{
+				ri->ri_count = 0;
+				++ri->ri_idx;
+			}
+			break;
+
+		default:
+			break;
+		}
+
+		li->li_isquarantined = LDAP_BACK_FQ_YES;
+		ri->ri_last = slap_get_time();
+
+	} else if ( li->li_isquarantined != LDAP_BACK_FQ_NO ) {
+		Debug( LDAP_DEBUG_ANY,
+			"%s: ldap_back_quarantine exit.\n",
+			op->o_log_prefix, ri->ri_idx, ri->ri_count );
+
+		ri->ri_count = 0;
+		ri->ri_idx = 0;
+		li->li_isquarantined = LDAP_BACK_FQ_NO;
+	}
+
+	if ( dolock ) {
+		ldap_pvt_thread_mutex_unlock( &li->li_conninfo.lai_mutex );
+	}
+}
+
 /*
  * ldap_back_dobind
  *
@@ -929,7 +1022,7 @@ retry_lock:;
 
 		if ( li->li_acl_secprops != NULL ) {
 			rc = ldap_set_option( lc->lc_ld,
-				LDAP_OPT_X_SASL_SECPROPS, li->li_acl_secprops);
+				LDAP_OPT_X_SASL_SECPROPS, li->li_acl_secprops );
 
 			if ( rc != LDAP_OPT_SUCCESS ) {
 				Debug( LDAP_DEBUG_ANY, "Error: ldap_set_option "
@@ -1012,6 +1105,11 @@ retry:;
 			}
 		}
 
+		if ( LDAP_BACK_QUARANTINE( li ) ) {
+			ldap_back_quarantine( op, rs, dolock );
+		}
+
+		/* FIXME: one binding-- too many? */
 		lc->lc_binding--;
 		ldap_back_freeconn( op, lc, dolock );
 		rs->sr_err = slap_map_api2result( rs );
@@ -1025,6 +1123,10 @@ retry:;
 	}
 
 done:;
+	if ( LDAP_BACK_QUARANTINE( li ) ) {
+		ldap_back_quarantine( op, rs, dolock );
+	}
+
 	lc->lc_binding--;
 	LDAP_BACK_CONN_BINDING_CLEAR( lc );
 	rc = LDAP_BACK_CONN_ISBOUND( lc );
@@ -1152,6 +1254,8 @@ ldap_back_op_result(
 		time_t			timeout,
 		ldap_back_send_t	sendok )
 {
+	ldapinfo_t	*li = (ldapinfo_t *)op->o_bd->be_private;
+
 	char		*match = NULL;
 	LDAPMessage	*res = NULL;
 	char		*text = NULL;
@@ -1247,6 +1351,9 @@ retry:;
 			rs->sr_matched = match;
 		}
 	}
+	if ( LDAP_BACK_QUARANTINE( li ) ) {
+		ldap_back_quarantine( op, rs, 1 );
+	}
 	if ( op->o_conn &&
 		( ( sendok & LDAP_BACK_SENDOK ) 
 			|| ( ( sendok & LDAP_BACK_SENDERR ) && rs->sr_err != LDAP_SUCCESS ) ) )
@@ -1307,11 +1414,15 @@ ldap_back_retry( ldapconn_t **lcp, Operation *op, SlapReply *rs, ldap_back_send_
 		rc = ldap_back_prepare_conn( lcp, op, rs, sendok );
 		if ( rc != LDAP_SUCCESS ) {
 			rc = 0;
+			/* freeit, because lc_refcnt == 1 */
+			(void)ldap_back_conn_free( *lcp );
 			*lcp = NULL;
 
 		} else {
 			rc = ldap_back_dobind_int( *lcp, op, rs, sendok, 0, 0 );
-			if ( rc == 0 ) {
+			if ( rc == 0 && *lcp != NULL ) {
+				/* freeit, because lc_refcnt == 1 */
+				(void)ldap_back_conn_free( *lcp );
 				*lcp = NULL;
 			}
 		}

servers/slapd/back-ldap/config.c

@@ -66,6 +66,7 @@ enum {
 	LDAP_BACK_CFG_VERSION,
 	LDAP_BACK_CFG_SINGLECONN,
 	LDAP_BACK_CFG_CANCEL,
+	LDAP_BACK_CFG_QUARANTINE,
 	LDAP_BACK_CFG_REWRITE,
 
 	LDAP_BACK_CFG_LAST
@@ -268,6 +269,14 @@ static ConfigTable ldapcfg[] = {
 			"SYNTAX OMsDirectoryString "
 			"SINGLE-VALUE )",
 		NULL, NULL },
+	{ "quarantine", "retrylist", 2, 0, 0,
+		ARG_MAGIC|LDAP_BACK_CFG_QUARANTINE,
+		ldap_back_cf_gen, "( OLcfgDbAt:3.21 "
+			"NAME 'olcDbQuarantine' "
+			"DESC 'Quarantine database if connection fails and retry according to rule' "
+			"SYNTAX OMsDirectoryString "
+			"SINGLE-VALUE )",
+		NULL, NULL },
 	{ "suffixmassage", "[virtual]> <real", 2, 3, 0,
 		ARG_STRING|ARG_MAGIC|LDAP_BACK_CFG_REWRITE,
 		ldap_back_cf_gen, NULL, NULL, NULL },
@@ -304,6 +313,7 @@ static ConfigOCs ldapocs[] = {
 			"$ olcDbIdleTimeout "
 			"$ olcDbSingleConn "
 			"$ olcDbCancel "
+			"$ olcDbQuarantine "
 		") )",
 		 	Cft_Database, ldapcfg},
 	{ NULL, 0, NULL }
@@ -349,16 +359,161 @@ static slap_cf_aux_table timeout_table[] = {
 	{ BER_BVNULL, 0, 0, 0, NULL }
 };
 
+int
+slap_retry_info_parse(
+	char			*in,
+	slap_retry_info_t 	*ri,
+	char			*buf,
+	ber_len_t		buflen )
+{
+	char			**retrylist = NULL;
+	int			rc = 0;
+	int			i;
+
+	slap_str2clist( &retrylist, in, " ;" );
+	if ( retrylist == NULL ) {
+		return 1;
+	}
+
+	for ( i = 0; retrylist[ i ] != NULL; i++ )
+		/* count */ ;
+
+	ri->ri_interval = ch_calloc( sizeof( time_t ), i + 1 );
+	ri->ri_num = ch_calloc( sizeof( int ), i + 1 );
+
+	for ( i = 0; retrylist[ i ] != NULL; i++ ) {
+		char	*sep = strchr( retrylist[ i ], ',' );
+
+		if ( sep == NULL ) {
+			snprintf( buf, buflen,
+				"missing comma in retry pattern #%d \"%s\"",
+				i, retrylist[ i ] );
+			rc = 1;
+			goto done;
+		}
+
+		*sep++ = '\0';
+
+		if ( lutil_atol( &ri->ri_interval[ i ], retrylist[ i ] ) ) {
+			snprintf( buf, buflen,
+				"unable to parse interval #%d \"%s\"",
+				i, retrylist[ i ] );
+			rc = 1;
+			goto done;
+		}
+
+		if ( strcmp( sep, "+" ) == 0 ) {
+			if ( retrylist[ i + 1 ] != NULL ) {
+				snprintf( buf, buflen,
+					"extra cruft after retry pattern "
+					"#%d \"%s,+\" with \"forever\" mark",
+					i, retrylist[ i ] );
+				rc = 1;
+				goto done;
+			}
+			ri->ri_num[ i ] = SLAP_RETRYNUM_FOREVER;
+			
+		} else if ( lutil_atoi( &ri->ri_num[ i ], sep ) ) {
+			snprintf( buf, buflen,
+				"unable to parse retry num #%d \"%s\"",
+				i, sep );
+			rc = 1;
+			goto done;
+		}
+	}
+
+	ri->ri_num[ i ] = SLAP_RETRYNUM_TAIL;
+
+	ri->ri_idx = 0;
+	ri->ri_count = 0;
+	ri->ri_last = (time_t)(-1);
+
+done:;
+	ldap_charray_free( retrylist );
+
+	if ( rc ) {
+		slap_retry_info_destroy( ri );
+	}
+
+	return rc;
+}
+
+int
+slap_retry_info_unparse(
+	slap_retry_info_t	*ri,
+	struct berval		*bvout )
+{
+	int		i;
+	char		buf[ BUFSIZ * 2 ],
+			*ptr = buf;
+	struct berval	bv = BER_BVNULL;
+
+	assert( ri != NULL );
+	assert( bvout != NULL );
+
+	BER_BVZERO( bvout );
+
+#define WHATSLEFT	( sizeof( buf ) - ( ptr - buf ) )
+
+	for ( i = 0; ri->ri_num[ i ] != SLAP_RETRYNUM_TAIL; i++ ) {
+		if ( i > 0 ) {
+			if ( WHATSLEFT <= 1 ) {
+				return 1;
+			}
+			*ptr++ = ';';
+		}
+
+		ptr += snprintf( ptr, WHATSLEFT, "%ld,", (long)ri->ri_interval[i] );
+		if ( WHATSLEFT <= 0 ) {
+			return 1;
+		}
+
+		if ( ri->ri_num[i] == SLAP_RETRYNUM_FOREVER ) {
+			if ( WHATSLEFT <= 1 ) {
+				return 1;
+			}
+			*ptr++ = '+';
+
+		} else {
+			ptr += snprintf( ptr, WHATSLEFT, "%d", ri->ri_num[i] );
+			if ( WHATSLEFT <= 0 ) {
+				return 1;
+			}
+		}
+	}
+
+	bv.bv_val = buf;
+	bv.bv_len = ptr - buf;
+
+	ber_dupbv( bvout, &bv );
+
+	return 0;
+}
+
+void
+slap_retry_info_destroy(
+	slap_retry_info_t	*ri )
+{
+	assert( ri != NULL );
+
+	assert( ri->ri_interval != NULL );
+	ch_free( ri->ri_interval );
+	ri->ri_interval = NULL;
+
+	assert( ri->ri_num != NULL );
+	ch_free( ri->ri_num );
+	ri->ri_num = NULL;
+}
+
 static int
 ldap_back_cf_gen( ConfigArgs *c )
 {
 	ldapinfo_t	*li = ( ldapinfo_t * )c->be->be_private;
-	int		rc;
+	int		rc = 0;
 	int		i;
 
 	if ( c->op == SLAP_CONFIG_EMIT ) {
 		struct berval	bv = BER_BVNULL;
-		rc = 0;
 
 		if ( li == NULL ) {
 			return 1;
@@ -678,6 +833,18 @@ ldap_back_cf_gen( ConfigArgs *c )
 			}
 			} break;
 
+		case LDAP_BACK_CFG_QUARANTINE:
+			if ( !LDAP_BACK_QUARANTINE( li ) ) {
+				rc = 1;
+				break;
+			}
+
+			rc = slap_retry_info_unparse( &li->li_quarantine, &bv );
+			if ( rc == 0 ) {
+				ber_bvarray_add( &c->rvalue_vals, &bv );
+			}
+			break;
+
 		default:
 			/* FIXME: we need to handle all... */
 			assert( 0 );
@@ -686,7 +853,6 @@ ldap_back_cf_gen( ConfigArgs *c )
 		return rc;
 
 	} else if ( c->op == LDAP_MOD_DELETE ) {
-		rc = 0;
 		switch( c->type ) {
 		case LDAP_BACK_CFG_URI:
 			if ( li->li_uri != NULL ) {
@@ -776,6 +942,15 @@ ldap_back_cf_gen( ConfigArgs *c )
 			li->li_flags &= ~LDAP_BACK_F_SINGLECONN;
 			break;
 
+		case LDAP_BACK_CFG_QUARANTINE:
+			if ( !LDAP_BACK_QUARANTINE( li ) ) {
+				break;
+			}
+
+			slap_retry_info_destroy( &li->li_quarantine );
+			li->li_isquarantined = 0;
+			break;
+
 		default:
 			/* FIXME: we need to handle all... */
 			assert( 0 );
@@ -1461,6 +1636,23 @@ done_url:;
 		li->li_flags |= mask;
 		} break;
 
+	case LDAP_BACK_CFG_QUARANTINE:
+		if ( LDAP_BACK_QUARANTINE( li ) ) {
+			snprintf( c->msg, sizeof( c->msg ),
+				"quarantine already defined" );
+			Debug( LDAP_DEBUG_ANY, "%s: %s.\n", c->log, c->msg, 0 );
+			return 1;
+		}
+		rc = slap_retry_info_parse( c->argv[1], &li->li_quarantine,
+			c->msg, sizeof( c->msg ) );
+		if ( rc ) {
+			Debug( LDAP_DEBUG_ANY, "%s: %s.\n", c->log, c->msg, 0 );
+		}
+		/* give it a chance to retry if the pattern gets reset
+		 * via back-config */
+		li->li_isquarantined = 0;
+		break;
+
 	case LDAP_BACK_CFG_REWRITE:
 		snprintf( c->msg, sizeof( c->msg ),
 			"rewrite/remap capabilities have been moved "
@@ -1476,7 +1668,7 @@ done_url:;
 		break;
 	}
 
-	return 0;
+	return rc;
 }
 
 int

servers/slapd/back-ldap/extended.c

@@ -106,6 +106,8 @@ ldap_back_exop_passwd(
 		Operation	*op,
 		SlapReply	*rs )
 {
+	ldapinfo_t	*li = (ldapinfo_t *) op->o_bd->be_private;
+
 	ldapconn_t	*lc;
 	req_pwdexop_s	*qpw = &op->oq_pwdexop;
 	LDAPMessage	*res;
@@ -181,10 +183,18 @@ retry:
 				goto retry;
 			}
 		}
+
+		if ( LDAP_BACK_QUARANTINE( li ) ) {
+			ldap_back_quarantine( op, rs, 1 );
+		}
+
 		if ( text ) rs->sr_text = text;
 		send_ldap_extended( op, rs );
 		/* otherwise frontend resends result */
 		rc = rs->sr_err = SLAPD_ABANDON;
+
+	} else if ( LDAP_BACK_QUARANTINE( li ) ) {
+		ldap_back_quarantine( op, rs, 1 );
 	}
 
 	/* these have to be freed anyway... */
@@ -210,6 +220,8 @@ ldap_back_exop_generic(
 	Operation	*op,
 	SlapReply	*rs )
 {
+	ldapinfo_t	*li = (ldapinfo_t *) op->o_bd->be_private;
+
 	ldapconn_t	*lc;
 	LDAPMessage	*res;
 	ber_int_t	msgid;
@@ -267,10 +279,18 @@ retry:
 				goto retry;
 			}
 		}
+
+		if ( LDAP_BACK_QUARANTINE( li ) ) {
+			ldap_back_quarantine( op, rs, 1 );
+		}
+
 		if ( text ) rs->sr_text = text;
 		send_ldap_extended( op, rs );
 		/* otherwise frontend resends result */
 		rc = rs->sr_err = SLAPD_ABANDON;
+
+	} else if ( LDAP_BACK_QUARANTINE( li ) ) {
+		ldap_back_quarantine( op, rs, 1 );
 	}
 
 	/* these have to be freed anyway... */

servers/slapd/back-ldap/init.c

@@ -317,6 +317,9 @@ ldap_back_db_destroy(
                	if ( li->li_conninfo.lai_tree ) {
 			avl_free( li->li_conninfo.lai_tree, ldap_back_conn_free );
 		}
+		if ( LDAP_BACK_QUARANTINE( li ) ) {
+			slap_retry_info_destroy( &li->li_quarantine );
+		}
 
 		ldap_pvt_thread_mutex_unlock( &li->li_conninfo.lai_mutex );
 		ldap_pvt_thread_mutex_destroy( &li->li_conninfo.lai_mutex );

servers/slapd/back-ldap/proto-ldap.h

@@ -77,6 +77,18 @@ ldap_back_proxy_authz_ctrl_free(
 		Operation	*op,
 		LDAPControl	***pctrls );
 
+extern void
+ldap_back_quarantine(
+	Operation	*op,
+	SlapReply	*rs,
+	int		dolock );
+
+extern void slap_retry_info_destroy( slap_retry_info_t *ri );
+extern int slap_retry_info_parse( char *in, slap_retry_info_t *ri,
+	char *buf, ber_len_t buflen );
+extern int slap_retry_info_unparse( slap_retry_info_t *ri, struct berval *bvout );
+
+
 extern int chain_initialize( void );
 #ifdef LDAP_DEVEL
 extern int distproc_initialize( void );

servers/slapd/back-ldap/search.c

@@ -141,6 +141,8 @@ ldap_back_search(
 		Operation	*op,
 		SlapReply	*rs )
 {
+	ldapinfo_t	*li = (ldapinfo_t *) op->o_bd->be_private;
+
 	ldapconn_t	*lc;
 	struct timeval	tv;
 	time_t		stoptime = (time_t)-1;
@@ -464,6 +466,10 @@ retry:
 	}
 
 finish:;
+	if ( LDAP_BACK_QUARANTINE( li ) ) {
+		ldap_back_quarantine( op, rs, 1 );
+	}
+
 	if ( rc != SLAPD_ABANDON ) {
 		send_ldap_result( op, rs );
 	}

servers/slapd/back-meta/add.c

@@ -220,6 +220,10 @@ retry:;
 
 	} else {
 		send_ldap_result( op, rs );
+
+		if ( META_BACK_QUARANTINE( mi ) ) {
+			meta_back_quarantine( op, rs, candidate, 1 );
+		}
 	}
 
 cleanup:;

servers/slapd/back-meta/back-meta.h

@@ -227,6 +227,9 @@ typedef struct metatarget_t {
 
 	struct ldaprwmap	mt_rwmap;
 
+	sig_atomic_t		mt_isquarantined;
+	slap_retry_info_t	mt_quarantine;
+
 	unsigned		mt_flags;
 #define	META_BACK_TGT_ISSET(mt,f)		( ( (mt)->mt_flags & (f) ) == (f) )
 #define	META_BACK_TGT_ISMASK(mt,m,f)		( ( (mt)->mt_flags & (m) ) == (f) )
@@ -276,6 +279,11 @@ typedef struct metainfo_t {
 	
 	ldap_avl_info_t		mi_conninfo;
 
+	/* NOTE: quarantine uses the connection mutex */
+	slap_retry_info_t	mi_quarantine;
+
+#define	META_BACK_QUARANTINE(mi)	( (mi)->mi_quarantine.ri_num != NULL )
+
 	unsigned		mi_flags;
 #define	li_flags		mi_flags
 /* uses flags as defined in <back-ldap/back-ldap.h> */
@@ -334,12 +342,18 @@ extern int
 meta_back_init_one_conn(
 	Operation		*op,
 	SlapReply		*rs,
-	metatarget_t		*mt, 
 	metaconn_t		*mc,
 	int			candidate,
 	int			ispriv,
 	ldap_back_send_t	sendok );
 
+extern void
+meta_back_quarantine(
+	Operation		*op,
+	SlapReply		*rs,
+	int			candidate,
+	int			dolock );
+
 extern int
 meta_back_single_bind(
 	Operation		*op,

servers/slapd/back-meta/bind.c

@@ -442,7 +442,7 @@ retry:;
 	}
 
 	if ( mi->mi_cache.ttl != META_DNCACHE_DISABLED
-			&& op->o_req_ndn.bv_len != 0 )
+			&& !BER_BVISEMPTY( &op->o_req_ndn ) )
 	{
 		( void )meta_dncache_update_entry( &mi->mi_cache,
 				&op->o_req_ndn, candidate );
@@ -453,6 +453,10 @@ return_results:;
 		free( mdn.bv_val );
 	}
 
+	if ( META_BACK_QUARANTINE( mi ) ) {
+		meta_back_quarantine( op, rs, candidate, 1 );
+	}
+
 	return rs->sr_err;
 }
 
@@ -594,7 +598,7 @@ retry:;
 				        /* mc here must be the regular mc,
 					 * reset and ready for init */
 				        rc = meta_back_init_one_conn( op, rs,
-						mt, mc, candidate,
+						mc, candidate,
 						LDAP_BACK_CONN_ISPRIV( mc ),
 						LDAP_BACK_DONTSEND );
 					if ( rc == LDAP_SUCCESS ) {
@@ -655,6 +659,10 @@ done:;
 		}
 	}
 
+	if ( META_BACK_QUARANTINE( mi ) ) {
+		meta_back_quarantine( op, rs, candidate, dolock );
+	}
+
 	return rc;
 }
 
@@ -981,6 +989,10 @@ meta_back_op_result(
 					( rmatch ? rmatch : "" ) );
 		}
 
+		if ( META_BACK_QUARANTINE( mi ) ) {
+			meta_back_quarantine( op, rs, candidate, 1 );
+		}
+
 	} else {
 		for ( i = 0; i < mi->mi_ntargets; i++ ) {
 			metasingleconn_t	*msc = &mc->mc_conns[ i ];
@@ -1051,6 +1063,10 @@ meta_back_op_result(
 					ldap_memfree( match );
 				}
 			}
+
+			if ( META_BACK_QUARANTINE( mi ) ) {
+				meta_back_quarantine( op, rs, i, 1 );
+			}
 		}
 	}
 	

servers/slapd/back-meta/config.c

@@ -171,6 +171,7 @@ meta_back_db_config(
 		mt->mt_urllist_p = mt;
 
 		mt->mt_nretries = mi->mi_nretries;
+		mt->mt_quarantine = mi->mi_quarantine;
 		mt->mt_flags = mi->mi_flags;
 		mt->mt_version = mi->mi_version;
 		mt->mt_network_timeout = mi->mi_network_timeout;
@@ -981,6 +982,49 @@ meta_back_db_config(
 			return 1;
 		}
 		ber_str2bv( argv[ 1 ], 0L, 1, &mi->mi_targets[ i ]->mt_pseudorootpw );
+
+	/* quarantine */
+	} else if ( strcasecmp( argv[ 0 ], "quarantine" ) == 0 ) {
+		char			buf[ SLAP_TEXT_BUFLEN ] = { '\0' };
+		slap_retry_info_t	*ri = mi->mi_ntargets ?
+				&mi->mi_targets[ mi->mi_ntargets - 1 ]->mt_quarantine
+				: &mi->mi_quarantine;
+
+		if ( META_BACK_QUARANTINE( mi ) ) {
+			Debug( LDAP_DEBUG_ANY,
+				"%s: line %d: quarantine already defined.\n",
+				fname, lineno, 0 );
+			return 1;
+		}
+
+		switch ( argc ) {
+		case 2:
+			break;
+
+		case 1:
+			Debug( LDAP_DEBUG_ANY,
+				"%s: line %d: missing arg in \"quarantine <pattern list>\".\n",
+				fname, lineno, 0 );
+			return 1;
+
+		default:
+			Debug( LDAP_DEBUG_ANY,
+				"%s: line %d: extra cruft after \"quarantine <pattern list>\".\n",
+				fname, lineno, 0 );
+			return 1;
+		}
+
+		if ( ri != &mi->mi_quarantine ) {
+			ri->ri_interval = NULL;
+			ri->ri_num = NULL;
+		}
+
+		if ( slap_retry_info_parse( argv[ 1 ], ri, buf, sizeof( buf ) ) ) {
+			Debug( LDAP_DEBUG_ANY,
+				"%s line %d: %s.\n",
+				fname, lineno, buf );
+			return 1;
+		}
 	
 	/* dn massaging */
 	} else if ( strcasecmp( argv[ 0 ], "suffixmassage" ) == 0 ) {

servers/slapd/back-meta/conn.c

@@ -256,13 +256,13 @@ int
 meta_back_init_one_conn(
 	Operation		*op,
 	SlapReply		*rs,
-	metatarget_t		*mt, 
 	metaconn_t		*mc,
 	int			candidate,
 	int			ispriv,
 	ldap_back_send_t	sendok )
 {
 	metainfo_t		*mi = ( metainfo_t * )op->o_bd->be_private;
+	metatarget_t		*mt = mi->mi_targets[ candidate ];
 	metasingleconn_t	*msc = &mc->mc_conns[ candidate ];
 	int			version;
 	dncookie		dc;
@@ -271,6 +271,35 @@ meta_back_init_one_conn(
 	int			is_ldaps = 0;
 #endif /* HAVE_TLS */
 
+	/* if the server is quarantined, and
+	 * - the current interval did not expire yet, or
+	 * - no more retries should occur,
+	 * don't return the connection */
+	if ( mt->mt_isquarantined ) {
+		slap_retry_info_t	*ri = &mt->mt_quarantine;
+		int			dont_retry = 1;
+
+		if ( mt->mt_isquarantined == LDAP_BACK_FQ_YES ) {
+			dont_retry = ( ri->ri_num[ ri->ri_idx ] == SLAP_RETRYNUM_TAIL
+				|| slap_get_time() < ri->ri_last + ri->ri_interval[ ri->ri_idx ] );
+			if ( !dont_retry ) {
+				Debug( LDAP_DEBUG_ANY,
+					"%s: meta_back_init_one_conn quarantine "
+					"retry block #%d try #%d.\n",
+					op->o_log_prefix, ri->ri_idx, ri->ri_count );
+				mt->mt_isquarantined = LDAP_BACK_FQ_RETRYING;
+			}
+		}
+
+		if ( dont_retry ) {
+			rs->sr_err = LDAP_UNAVAILABLE;
+			if ( op->o_conn && ( sendok & LDAP_BACK_SENDERR ) ) {
+				send_ldap_result( op, rs );
+			}
+			return rs->sr_err;
+		}
+	}
+
 	/*
 	 * Already init'ed
 	 */
@@ -542,7 +571,7 @@ meta_back_retry(
 		( void )rewrite_session_delete( mt->mt_rwmap.rwm_rw, op->o_conn );
 
 		/* mc here must be the regular mc, reset and ready for init */
-		rc = meta_back_init_one_conn( op, rs, mt, mc, candidate,
+		rc = meta_back_init_one_conn( op, rs, mc, candidate,
 			LDAP_BACK_CONN_ISPRIV( mc ), sendok );
 		if ( binding ) {
 			LDAP_BACK_CONN_BINDING_SET( msc );
@@ -592,6 +621,10 @@ meta_back_retry(
 		}
 	}
 
+	if ( META_BACK_QUARANTINE( mi ) ) {
+		meta_back_quarantine( op, rs, candidate, 0 );
+	}
+
 	ldap_pvt_thread_mutex_unlock( &mi->mi_conninfo.lai_mutex );
 
 	return rc == LDAP_SUCCESS ? 1 : 0;
@@ -937,15 +970,13 @@ retry_lock:
 		}
 
 		for ( i = 0; i < mi->mi_ntargets; i++ ) {
-			metatarget_t		*mt = mi->mi_targets[ i ];
-
 			/*
 			 * The target is activated; if needed, it is
 			 * also init'd
 			 */
 			candidates[ i ].sr_err = meta_back_init_one_conn( op,
-				rs, mt, mc, i,
+				rs, mc, i, LDAP_BACK_CONN_ISPRIV( &mc_curr ),
-				LDAP_BACK_CONN_ISPRIV( &mc_curr ), sendok );
+				sendok );
 			if ( candidates[ i ].sr_err == LDAP_SUCCESS ) {
 				candidates[ i ].sr_tag = META_CANDIDATE;
 				ncandidates++;
@@ -1102,7 +1133,7 @@ retry_lock2:;
 		 * also init'd. In case of error, meta_back_init_one_conn
 		 * sends the appropriate result.
 		 */
-		err = meta_back_init_one_conn( op, rs, mt, mc, i,
+		err = meta_back_init_one_conn( op, rs, mc, i,
 			LDAP_BACK_CONN_ISPRIV( &mc_curr ), sendok );
 		if ( err != LDAP_SUCCESS ) {
 			/*
@@ -1161,10 +1192,8 @@ retry_lock2:;
 				 * The target is activated; if needed, it is
 				 * also init'd
 				 */
-				int lerr = meta_back_init_one_conn( op, rs,
+				int lerr = meta_back_init_one_conn( op, rs, mc, i,
-						mt, mc, i,
+					LDAP_BACK_CONN_ISPRIV( &mc_curr ), LDAP_BACK_DONTSEND );
-						LDAP_BACK_CONN_ISPRIV( &mc_curr ),
-						sendok );
 				if ( lerr == LDAP_SUCCESS ) {
 					candidates[ i ].sr_tag = META_CANDIDATE;
 					candidates[ i ].sr_err = LDAP_SUCCESS;
@@ -1190,6 +1219,22 @@ retry_lock2:;
 					Debug( LDAP_DEBUG_ANY, "%s: meta_back_getconn[%d] failed: %d\n",
 						op->o_log_prefix, i, lerr );
 
+					if ( META_BACK_ONERR_STOP( mi ) ) {
+						if ( sendok & LDAP_BACK_SENDERR ) {
+							send_ldap_result( op, rs );
+							rs->sr_text = NULL;
+						}
+						if ( new_conn ) {
+							meta_back_freeconn( op, mc );
+
+						} else {
+							meta_back_release_conn( op, mc );
+						}
+
+						return NULL;
+					}
+
+					rs->sr_text = NULL;
 					continue;
 				}
 
@@ -1330,3 +1375,67 @@ meta_back_release_conn_lock(
 		ldap_pvt_thread_mutex_unlock( &mi->mi_conninfo.lai_mutex );
 	}
 }
+
+void
+meta_back_quarantine(
+	Operation	*op,
+	SlapReply	*rs,
+	int		candidate,
+	int		dolock )
+{
+	metainfo_t		*mi = (metainfo_t *)op->o_bd->be_private;
+	metatarget_t		*mt = mi->mi_targets[ candidate ];
+
+	slap_retry_info_t	*ri = &mt->mt_quarantine;
+
+	if ( dolock ) {
+		ldap_pvt_thread_mutex_lock( &mi->mi_conninfo.lai_mutex );
+	}
+
+	if ( rs->sr_err == LDAP_UNAVAILABLE ) {
+		switch ( mt->mt_isquarantined ) {
+		case LDAP_BACK_FQ_NO:
+			Debug( LDAP_DEBUG_ANY,
+				"%s: meta_back_quarantine enter.\n",
+				op->o_log_prefix, 0, 0 );
+
+			ri->ri_idx = 0;
+			ri->ri_count = 0;
+			break;
+
+		case LDAP_BACK_FQ_RETRYING:
+			Debug( LDAP_DEBUG_ANY,
+				"%s: meta_back_quarantine block #%d try #%d failed.\n",
+				op->o_log_prefix, ri->ri_idx, ri->ri_count );
+
+			++ri->ri_count;
+			if ( ri->ri_num[ ri->ri_idx ] != SLAP_RETRYNUM_FOREVER
+				&& ri->ri_count == ri->ri_num[ ri->ri_idx ] )
+			{
+				ri->ri_count = 0;
+				++ri->ri_idx;
+			}
+			break;
+
+		default:
+			break;
+		}
+
+		mt->mt_isquarantined = LDAP_BACK_FQ_YES;
+		ri->ri_last = slap_get_time();
+
+	} else if ( mt->mt_isquarantined != LDAP_BACK_FQ_NO ) {
+		Debug( LDAP_DEBUG_ANY,
+			"%s: meta_back_quarantine exit.\n",
+			op->o_log_prefix, ri->ri_idx, ri->ri_count );
+
+		ri->ri_count = 0;
+		ri->ri_idx = 0;
+		mt->mt_isquarantined = LDAP_BACK_FQ_NO;
+	}
+
+	if ( dolock ) {
+		ldap_pvt_thread_mutex_unlock( &mi->mi_conninfo.lai_mutex );
+	}
+}
+

servers/slapd/back-meta/delete.c

@@ -119,6 +119,10 @@ retry:;
 
 	} else {
 		send_ldap_result( op, rs );
+
+		if ( META_BACK_QUARANTINE( mi ) ) {
+			meta_back_quarantine( op, rs, candidate, 1 );
+		}
 	}
 
 cleanup:;

servers/slapd/back-meta/init.c

@@ -267,6 +267,12 @@ meta_back_db_destroy(
 		 */
 		if ( mi->mi_targets != NULL ) {
 			for ( i = 0; i < mi->mi_ntargets; i++ ) {
+				if ( META_BACK_QUARANTINE( mi )
+					&& mi->mi_targets[ i ]->mt_quarantine.ri_num != mi->mi_quarantine.ri_num )
+				{
+					slap_retry_info_destroy( &mi->mi_targets[ i ]->mt_quarantine );
+				}
+
 				target_free( mi->mi_targets[ i ] );
 			}
 
@@ -287,6 +293,10 @@ meta_back_db_destroy(
 		if ( mi->mi_candidates != NULL ) {
 			ber_memfree_x( mi->mi_candidates, NULL );
 		}
+
+		if ( META_BACK_QUARANTINE( mi ) ) {
+			slap_retry_info_destroy( &mi->mi_quarantine );
+		}
 	}
 
 	free( be->be_private );

servers/slapd/back-meta/modify.c

@@ -231,6 +231,10 @@ cleanup:;
 
 	} else {
 		send_ldap_result( op, rs );
+
+		if ( META_BACK_QUARANTINE( mi ) ) {
+			meta_back_quarantine( op, rs, candidate, 1 );
+		}
 	}
 
 done:;

servers/slapd/back-meta/modrdn.c

@@ -175,6 +175,10 @@ cleanup:;
 
 	} else {
 		send_ldap_result( op, rs );
+
+		if ( META_BACK_QUARANTINE( mi ) ) {
+			meta_back_quarantine( op, rs, candidate, 1 );
+		}
 	}
 
 done:;

servers/slapd/back-meta/search.c

@@ -739,8 +739,7 @@ really_bad:;
 				 */
 				candidates[ i ].sr_msgid = META_MSGID_IGNORE;
 				--ncandidates;
-				rs->sr_err = candidates[ i ].sr_err = LDAP_OTHER;
+				rs->sr_err = candidates[ i ].sr_err;
-				rs->sr_text = "remote server unavailable";
 
 			} else if ( rc == LDAP_RES_SEARCH_ENTRY ) {
 				if ( candidates[ i ].sr_type == REP_INTERMEDIATE ) {
@@ -1258,6 +1257,10 @@ finish:;
 			ldap_controls_free( candidates[ i ].sr_ctrls );
 			candidates[ i ].sr_ctrls = NULL;
 		}
+
+		if ( META_BACK_QUARANTINE( mi ) ) {
+			meta_back_quarantine( op, &candidates[ i ], i, 1 );
+		}
 	}
 
 	if ( mc ) {