- Ratelimit does not apply to prefetched queries, and ratelimit-factor

is default 10.  Repeated normal queries get resolved and with
  prefetch stay in the cache.


git-svn-id: file:///svn/unbound/trunk@3399 be551aaa-1e26-0410-a405-d3ace91eadb9

Makefile.in

@@ -642,8 +642,8 @@ iterator.lo iterator.o: $(srcdir)/iterator/iterator.c config.h $(srcdir)/iterato
  $(srcdir)/validator/val_neg.h $(srcdir)/services/cache/dns.h $(srcdir)/services/cache/infra.h \
  $(srcdir)/util/rtt.h $(srcdir)/util/netevent.h $(srcdir)/util/net_help.h $(srcdir)/util/regional.h \
  $(srcdir)/util/data/dname.h $(srcdir)/util/data/msgencode.h $(srcdir)/util/fptr_wlist.h $(srcdir)/util/tube.h \
- $(srcdir)/services/mesh.h $(srcdir)/services/modstack.h $(srcdir)/util/config_file.h $(srcdir)/sldns/wire2str.h \
- $(srcdir)/sldns/parseutil.h $(srcdir)/sldns/sbuffer.h
+ $(srcdir)/services/mesh.h $(srcdir)/services/modstack.h $(srcdir)/util/config_file.h $(srcdir)/util/random.h \
+ $(srcdir)/sldns/wire2str.h $(srcdir)/sldns/parseutil.h $(srcdir)/sldns/sbuffer.h
 iter_delegpt.lo iter_delegpt.o: $(srcdir)/iterator/iter_delegpt.c config.h $(srcdir)/iterator/iter_delegpt.h \
  $(srcdir)/util/log.h $(srcdir)/services/cache/dns.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h \
  $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/regional.h \

doc/Changelog

@@ -1,5 +1,8 @@
 16 April 2015: Wouter
 	- Add local-zone type inform_deny, that logs query and drops answer.
+	- Ratelimit does not apply to prefetched queries, and ratelimit-factor
+	  is default 10.  Repeated normal queries get resolved and with
+	  prefetch stay in the cache.
 
 10 April 2015: Wouter
 	- unbound-control ratelimit_list lists high rate domains.

doc/example.conf.in

@@ -561,6 +561,9 @@ server:
 	# ratelimit-size: 4m
 	# ratelimit cache slabs, reduces lock contention if equal to cpucount.
 	# ratelimit-slabs: 4
+	
+	# 0 blocks when ratelimited, otherwise let 1/xth traffic through
+	# ratelimit-factor: 10
 
 	# override the ratelimit for a specific domain name.
 	# give this setting multiple times to have multiple overrides.

doc/unbound.conf.5.in

@@ -1006,6 +1006,15 @@ Give power of 2 number of slabs, this is used to reduce lock contention
 in the ratelimit tracking data structure.  Close to the number of cpus is
 a fairly good setting.
 .TP 5
+.B ratelimit\-factor: \fI<number>
+Set the amount of queries to rate limit when the limit is exceeded.
+If set to 0, all queries are dropped for domains where the limit is
+exceeded.  If set to another value, 1 in that number is allowed through
+to complete.  Default is 10, allowing 1/10 traffic to flow normally.
+This can make ordinary queries complete (if repeatedly queried for),
+and enter the cache, whilst also mitigiting the traffic flow by the
+factor given.
+.TP 5
 .B ratelimit\-for\-domain: \fI<domain> <number qps>
 Override the global ratelimit for an exact match domain name with the listed
 number.  You can give this for any number of names.  For example, for

iterator/iterator.c

@@ -61,6 +61,7 @@
 #include "util/data/msgencode.h"
 #include "util/fptr_wlist.h"
 #include "util/config_file.h"
+#include "util/random.h"
 #include "sldns/rrdef.h"
 #include "sldns/wire2str.h"
 #include "sldns/parseutil.h"
@@ -120,6 +121,7 @@ iter_new(struct module_qstate* qstate, int id)
 	iq->query_restart_count = 0;
 	iq->referral_count = 0;
 	iq->sent_count = 0;
+	iq->ratelimit_ok = 0;
 	iq->target_count = NULL;
 	iq->wait_priming_stub = 0;
 	iq->refetch_glue = 0;
@@ -1125,17 +1127,31 @@ processInitRequest(struct module_qstate* qstate, struct iter_qstate* iq,
 			 * results of priming. */
 			return 0;
 		}
-		if(infra_ratelimit_exceeded(qstate->env->infra_cache,
-			iq->dp->name, iq->dp->namelen, *qstate->env->now)) {
+		if(!iq->ratelimit_ok && qstate->prefetch_leeway)
+			iq->ratelimit_ok = 1; /* allow prefetches, this keeps
+			otherwise valid data in the cache */
+		if(!iq->ratelimit_ok && infra_ratelimit_exceeded(
+			qstate->env->infra_cache, iq->dp->name,
+			iq->dp->namelen, *qstate->env->now)) {
 			/* and increment the rate, so that the rate for time
 			 * now will also exceed the rate, keeping cache fresh */
 			(void)infra_ratelimit_inc(qstate->env->infra_cache,
 				iq->dp->name, iq->dp->namelen,
 				*qstate->env->now);
-			log_nametypeclass(VERB_ALGO, "ratelimit exceeded with "
-				"delegation point", iq->dp->name,
-				LDNS_RR_TYPE_NS, LDNS_RR_CLASS_IN);
-			return error_response(qstate, id, LDNS_RCODE_SERVFAIL);
+			/* see if we are passed through with slip factor */
+			if(qstate->env->cfg->ratelimit_factor != 0 &&
+				ub_random_max(qstate->env->rnd,
+				    qstate->env->cfg->ratelimit_factor) == 1) {
+				iq->ratelimit_ok = 1;
+				log_nametypeclass(VERB_ALGO, "ratelimit allowed through for "
+					"delegation point", iq->dp->name,
+					LDNS_RR_TYPE_NS, LDNS_RR_CLASS_IN);
+			} else {
+				log_nametypeclass(VERB_ALGO, "ratelimit exceeded with "
+					"delegation point", iq->dp->name,
+					LDNS_RR_TYPE_NS, LDNS_RR_CLASS_IN);
+				return error_response(qstate, id, LDNS_RCODE_SERVFAIL);
+			}
 		}
 
 		/* see if this dp not useless.
@@ -1927,7 +1943,7 @@ processQueryTargets(struct module_qstate* qstate, struct iter_qstate* iq,
 	}
 
 	/* if not forwarding, check ratelimits per delegationpoint name */
-	if(!(iq->chase_flags & BIT_RD)) {
+	if(!(iq->chase_flags & BIT_RD) && !iq->ratelimit_ok) {
 		if(!infra_ratelimit_inc(qstate->env->infra_cache, iq->dp->name,
 			iq->dp->namelen, *qstate->env->now)) {
 			verbose(VERB_ALGO, "query exceeded ratelimits");
@@ -1954,7 +1970,7 @@ processQueryTargets(struct module_qstate* qstate, struct iter_qstate* iq,
 	if(!outq) {
 		log_addr(VERB_DETAIL, "error sending query to auth server", 
 			&target->addr, target->addrlen);
-		if(!(iq->chase_flags & BIT_RD))
+		if(!(iq->chase_flags & BIT_RD) && !iq->ratelimit_ok)
 		    infra_ratelimit_dec(qstate->env->infra_cache, iq->dp->name,
 			iq->dp->namelen, *qstate->env->now);
 		return next_state(iq, QUERYTARGETS_STATE);
@@ -2107,7 +2123,7 @@ processQueryResponse(struct module_qstate* qstate, struct iter_qstate* iq,
 		 * delegation point, and back to the QUERYTARGETS_STATE. */
 		verbose(VERB_DETAIL, "query response was REFERRAL");
 
-		if(!(iq->chase_flags & BIT_RD)) {
+		if(!(iq->chase_flags & BIT_RD) && !iq->ratelimit_ok) {
 			/* we have a referral, no ratelimit, we can send
 			 * our queries to the given name */
 			infra_ratelimit_dec(qstate->env->infra_cache,

iterator/iterator.h

@@ -259,6 +259,9 @@ struct iter_qstate {
 	 * subqueries, the malloced-array is shared, [0] refcount. */
 	int* target_count;
 
+	/** if true, already tested for ratelimiting and passed the test */
+	int ratelimit_ok;
+
 	/**
 	 * The query must store NS records from referrals as parentside RRs
 	 * Enabled once it hits resolution problems, to throttle retries.

util/config_file.c

@@ -234,6 +234,7 @@ config_create(void)
 	cfg->ratelimit_size = 4*1024*1024;
 	cfg->ratelimit_for_domain = NULL;
 	cfg->ratelimit_below_domain = NULL;
+	cfg->ratelimit_factor = 10;
 	return cfg;
 error_exit:
 	config_delete(cfg); 
@@ -460,6 +461,7 @@ int config_set_option(struct config_file* cfg, const char* opt,
 	}
 	else S_MEMSIZE("ratelimit-size:", ratelimit_size)
 	else S_POW2("ratelimit-slabs:", ratelimit_slabs)
+	else S_NUMBER_OR_ZERO("ratelimit-factor:", ratelimit_factor)
 	/* val_sig_skew_min and max are copied into val_env during init,
 	 * so this does not update val_env with set_option */
 	else if(strcmp(opt, "val-sig-skew-min:") == 0)
@@ -728,6 +730,7 @@ config_get_option(struct config_file* cfg, const char* opt,
 	else O_DEC(opt, "ratelimit-slabs", ratelimit_slabs)
 	else O_LS2(opt, "ratelimit-for-domain", ratelimit_for_domain)
 	else O_LS2(opt, "ratelimit-below-domain", ratelimit_below_domain)
+	else O_DEC(opt, "ratelimit-factor", ratelimit_factor)
 	else O_DEC(opt, "val-sig-skew-min", val_sig_skew_min)
 	else O_DEC(opt, "val-sig-skew-max", val_sig_skew_max)
 	/* not here:

util/config_file.h

@@ -352,10 +352,12 @@ struct config_file {
 	size_t ratelimit_slabs;
 	/** memory size in bytes for ratelimit cache */
 	size_t ratelimit_size;
-	/* ratelimits for domain (exact match) */
+	/** ratelimits for domain (exact match) */
 	struct config_str2list* ratelimit_for_domain;
-	/* ratelimits below domain */
+	/** ratelimits below domain */
 	struct config_str2list* ratelimit_below_domain;
+	/** ratelimit factor, 0 blocks all, 10 allows 1/10 of traffic */
+	int ratelimit_factor;
 };
 
 /** from cfg username, after daemonise setup performed */

util/configlexer.lex

@@ -355,6 +355,7 @@ ratelimit-slabs{COLON}		{ YDVAR(1, VAR_RATELIMIT_SLABS) }
 ratelimit-size{COLON}		{ YDVAR(1, VAR_RATELIMIT_SIZE) }
 ratelimit-for-domain{COLON}	{ YDVAR(2, VAR_RATELIMIT_FOR_DOMAIN) }
 ratelimit-below-domain{COLON}	{ YDVAR(2, VAR_RATELIMIT_BELOW_DOMAIN) }
+ratelimit-factor{COLON}		{ YDVAR(1, VAR_RATELIMIT_FACTOR) }
 <INITIAL,val>{NEWLINE}		{ LEXOUT(("NL\n")); cfg_parser->line++; }
 
 	/* Quoted strings. Strip leading and ending quotes */

util/configparser.h

@@ -200,7 +200,8 @@ extern int yydebug;
      VAR_RATELIMIT_SLABS = 409,
      VAR_RATELIMIT_SIZE = 410,
      VAR_RATELIMIT_FOR_DOMAIN = 411,
-     VAR_RATELIMIT_BELOW_DOMAIN = 412
+     VAR_RATELIMIT_BELOW_DOMAIN = 412,
+     VAR_RATELIMIT_FACTOR = 413
    };
 #endif
 /* Tokens.  */
@@ -359,6 +360,7 @@ extern int yydebug;
 #define VAR_RATELIMIT_SIZE 410
 #define VAR_RATELIMIT_FOR_DOMAIN 411
 #define VAR_RATELIMIT_BELOW_DOMAIN 412
+#define VAR_RATELIMIT_FACTOR 413
 
 
 
@@ -372,7 +374,7 @@ typedef union YYSTYPE
 
 
 /* Line 2058 of yacc.c  */
-#line 376 "util/configparser.h"
+#line 378 "util/configparser.h"
 } YYSTYPE;
 # define YYSTYPE_IS_TRIVIAL 1
 # define yystype YYSTYPE /* obsolescent; will be withdrawn */

util/configparser.y

@@ -120,7 +120,7 @@ extern struct config_parser_state* cfg_parser;
 %token VAR_DNSTAP_LOG_FORWARDER_RESPONSE_MESSAGES
 %token VAR_HARDEN_ALGO_DOWNGRADE VAR_IP_TRANSPARENT
 %token VAR_RATELIMIT VAR_RATELIMIT_SLABS VAR_RATELIMIT_SIZE
-%token VAR_RATELIMIT_FOR_DOMAIN VAR_RATELIMIT_BELOW_DOMAIN
+%token VAR_RATELIMIT_FOR_DOMAIN VAR_RATELIMIT_BELOW_DOMAIN VAR_RATELIMIT_FACTOR
 
 %%
 toplevelvars: /* empty */ | toplevelvars toplevelvar ;
@@ -183,7 +183,7 @@ content_server: server_num_threads | server_verbosity | server_port |
 	server_infra_cache_min_rtt | server_harden_algo_downgrade |
 	server_ip_transparent | server_ratelimit | server_ratelimit_slabs |
 	server_ratelimit_size | server_ratelimit_for_domain |
-	server_ratelimit_below_domain
+	server_ratelimit_below_domain | server_ratelimit_factor
 	;
 stubstart: VAR_STUB_ZONE
 	{
@@ -1281,6 +1281,15 @@ server_ratelimit_below_domain: VAR_RATELIMIT_BELOW_DOMAIN STRING_ARG STRING_ARG
 		}
 	}
 	;
+server_ratelimit_factor: VAR_RATELIMIT_FACTOR STRING_ARG 
+	{ 
+		OUTYY(("P(server_ratelimit_factor:%s)\n", $2)); 
+		if(atoi($2) == 0 && strcmp($2, "0") != 0)
+			yyerror("number expected");
+		else cfg_parser->cfg->ratelimit_factor = atoi($2);
+		free($2);
+	}
+	;
 stub_name: VAR_NAME STRING_ARG
 	{
 		OUTYY(("P(name:%s)\n", $2));