extended statistics.

git-svn-id: file:///svn/unbound/trunk@1239 be551aaa-1e26-0410-a405-d3ace91eadb9

daemon/daemon.c

@@ -185,6 +185,9 @@ daemon_init()
 		free(daemon);
 		return NULL;
 	}
+	if(gettimeofday(&daemon->time_boot, NULL) < 0)
+		log_err("gettimeofday: %s", strerror(errno));
+	daemon->time_last_stat = daemon->time_boot;
 	return daemon;	
 }
 

daemon/daemon.h

@@ -90,6 +90,10 @@ struct daemon {
 	struct acl_list* acl;
 	/** local authority zones */
 	struct local_zones* local_zones;
+	/** last time of statistics printout */
+	struct timeval time_last_stat;
+	/** time when daemon started */
+	struct timeval time_boot;
 };
 
 /**

daemon/remote.c

@@ -50,7 +50,12 @@
 #include "util/log.h"
 #include "util/config_file.h"
 #include "util/net_help.h"
+#include "util/module.h"
 #include "services/listen_dnsport.h"
+#include "services/cache/rrset.h"
+#include "services/mesh.h"
+#include "util/storage/slabhash.h"
+#include "util/fptr_wlist.h"
 
 #ifdef HAVE_SYS_TYPES_H
 #  include <sys/types.h>
@@ -59,6 +64,16 @@
 #include <netdb.h>
 #endif
 
+/* just for portability */
+#ifdef SQ
+#undef SQ
+#endif
+
+/** what to put on statistics lines between var and value, ": " or "=" */
+#define SQ "="
+/** if true, inhibits a lot of =0 lines from the stats output */
+static const int inhibit_zero = 1;
+
 /** log ssl crypto err */
 static void
 log_crypto_err(const char* str)
@@ -74,6 +89,20 @@ log_crypto_err(const char* str)
 	}
 }
 
+/** subtract timers and the values do not overflow or become negative */
+static void
+timeval_subtract(struct timeval* d, struct timeval* end, struct timeval* start)
+{
+#ifndef S_SPLINT_S
+	d->tv_sec = end->tv_sec - start->tv_sec;
+	while(end->tv_usec < start->tv_usec) {
+		end->tv_usec += 1000000;
+		d->tv_sec--;
+	}
+	d->tv_usec = end->tv_usec - start->tv_usec;
+#endif
+}
+
 /** divide sum of timers to get average */
 static void
 timeval_divide(struct timeval* avg, struct timeval* sum, size_t d)
@@ -524,33 +553,33 @@ static int
 print_stats(SSL* ssl, char* nm, struct stats_info* s)
 {
 	struct timeval avg;
-	if(!ssl_printf(ssl, "%s.num.queries: %u\n", nm, 
+	if(!ssl_printf(ssl, "%s.num.queries"SQ"%u\n", nm, 
 		(unsigned)s->svr.num_queries)) return 0;
-	if(!ssl_printf(ssl, "%s.num.cachehits: %u\n", nm, 
+	if(!ssl_printf(ssl, "%s.num.cachehits"SQ"%u\n", nm, 
 		(unsigned)(s->svr.num_queries 
 			- s->svr.num_queries_missed_cache))) return 0;
-	if(!ssl_printf(ssl, "%s.num.cachemiss: %u\n", nm, 
+	if(!ssl_printf(ssl, "%s.num.cachemiss"SQ"%u\n", nm, 
 		(unsigned)s->svr.num_queries_missed_cache)) return 0;
-	if(!ssl_printf(ssl, "%s.num.recursivereplies: %u\n", nm, 
+	if(!ssl_printf(ssl, "%s.num.recursivereplies"SQ"%u\n", nm, 
 		(unsigned)s->mesh_replies_sent)) return 0;
-	if(!ssl_printf(ssl, "%s.requestlist.avg: %g\n", nm,
+	if(!ssl_printf(ssl, "%s.requestlist.avg"SQ"%g\n", nm,
 		s->svr.num_queries_missed_cache?
 			(double)s->svr.sum_query_list_size/
 			s->svr.num_queries_missed_cache : 0.0)) return 0;
-	if(!ssl_printf(ssl, "%s.requestlist.max: %u\n", nm,
+	if(!ssl_printf(ssl, "%s.requestlist.max"SQ"%u\n", nm,
 		(unsigned)s->svr.max_query_list_size)) return 0;
-	if(!ssl_printf(ssl, "%s.requestlist.overwritten: %u\n", nm,
+	if(!ssl_printf(ssl, "%s.requestlist.overwritten"SQ"%u\n", nm,
 		(unsigned)s->mesh_jostled)) return 0;
-	if(!ssl_printf(ssl, "%s.requestlist.exceeded: %u\n", nm,
+	if(!ssl_printf(ssl, "%s.requestlist.exceeded"SQ"%u\n", nm,
 		(unsigned)s->mesh_dropped)) return 0;
-	if(!ssl_printf(ssl, "%s.requestlist.current.all: %u\n", nm,
+	if(!ssl_printf(ssl, "%s.requestlist.current.all"SQ"%u\n", nm,
 		(unsigned)s->mesh_num_states)) return 0;
-	if(!ssl_printf(ssl, "%s.requestlist.current.user: %u\n", nm,
+	if(!ssl_printf(ssl, "%s.requestlist.current.user"SQ"%u\n", nm,
 		(unsigned)s->mesh_num_reply_states)) return 0;
 	timeval_divide(&avg, &s->mesh_replies_sum_wait, s->mesh_replies_sent);
-	if(!ssl_printf(ssl, "%s.recursion.time.avg: %d.%6.6d\n", nm,
+	if(!ssl_printf(ssl, "%s.recursion.time.avg"SQ"%d.%6.6d\n", nm,
 		(int)avg.tv_sec, (int)avg.tv_usec)) return 0;
-	if(!ssl_printf(ssl, "%s.recursion.time.median: %g\n", nm, 
+	if(!ssl_printf(ssl, "%s.recursion.time.median"SQ"%g\n", nm, 
 		s->mesh_time_median)) return 0;
 	return 1;
 }
@@ -565,6 +594,178 @@ print_thread_stats(SSL* ssl, int i, struct stats_info* s)
 	return print_stats(ssl, nm, s);
 }
 
+/** print mem stats */
+static int
+print_mem(SSL* ssl, struct worker* worker, struct daemon* daemon)
+{
+	int m;
+	size_t msg, rrset, val, iter;
+#ifdef HAVE_SBRK
+	extern void* unbound_start_brk;
+	void* cur = sbrk(0);
+	if(!ssl_printf(ssl, "mem.total.sbrk"SQ"%u\n", 
+		(unsigned)(cur-unbound_start_brk))) return 0;
+#endif /* HAVE_SBRK */
+	msg = slabhash_get_mem(daemon->env->msg_cache);
+	rrset = slabhash_get_mem(&daemon->env->rrset_cache->table);
+	val=0;
+	iter=0;
+	m = modstack_find(&worker->env.mesh->mods, "validator");
+	if(m != -1) {
+		fptr_ok(fptr_whitelist_mod_get_mem(worker->env.mesh->
+			mods.mod[m]->get_mem));
+		val = (*worker->env.mesh->mods.mod[m]->get_mem)
+			(&worker->env, m);
+	}
+	m = modstack_find(&worker->env.mesh->mods, "iterator");
+	if(m != -1) {
+		fptr_ok(fptr_whitelist_mod_get_mem(worker->env.mesh->
+			mods.mod[m]->get_mem));
+		iter = (*worker->env.mesh->mods.mod[m]->get_mem)
+			(&worker->env, m);
+	}
+
+	if(!ssl_printf(ssl, "mem.cache.rrset"SQ"%u\n", (unsigned)rrset))
+		return 0;
+	if(!ssl_printf(ssl, "mem.cache.message"SQ"%u\n", (unsigned)msg))
+		return 0;
+	if(!ssl_printf(ssl, "mem.mod.iterator"SQ"%u\n", (unsigned)iter))
+		return 0;
+	if(!ssl_printf(ssl, "mem.mod.validator"SQ"%u\n", (unsigned)val))
+		return 0;
+	return 1;
+}
+
+/** print uptime stats */
+static int
+print_uptime(SSL* ssl, struct worker* worker)
+{
+	struct timeval now = *worker->env.now_tv;
+	struct timeval up, dt;
+	timeval_subtract(&up, &now, &worker->daemon->time_boot);
+	timeval_subtract(&dt, &now, &worker->daemon->time_last_stat);
+	worker->daemon->time_last_stat = now;
+	if(!ssl_printf(ssl, "time.now"SQ"%d.%6.6d\n", 
+		(unsigned)now.tv_sec, (unsigned)now.tv_usec)) return 0;
+	if(!ssl_printf(ssl, "time.up"SQ"%d.%6.6d\n", 
+		(unsigned)up.tv_sec, (unsigned)up.tv_usec)) return 0;
+	if(!ssl_printf(ssl, "time.elapsed"SQ"%d.%6.6d\n", 
+		(unsigned)dt.tv_sec, (unsigned)dt.tv_usec)) return 0;
+	return 1;
+}
+
+/** print extended stats */
+static int
+print_ext(SSL* ssl, struct stats_info* s)
+{
+	int i;
+	char nm[16];
+	const ldns_rr_descriptor* desc;
+	const ldns_lookup_table* lt;
+	/* TYPE */
+	for(i=0; i<STATS_QTYPE_NUM; i++) {
+		if(inhibit_zero && s->svr.qtype[i] == 0)
+			continue;
+		desc = ldns_rr_descript((uint16_t)i);
+		if(desc && desc->_name) {
+			snprintf(nm, sizeof(nm), "%s", desc->_name);
+		} else {
+			snprintf(nm, sizeof(nm), "TYPE%d", i);
+		}
+		if(!ssl_printf(ssl, "num.query.type.%s"SQ"%u\n", 
+			nm, (unsigned)s->svr.qtype[i])) return 0;
+	}
+	if(!inhibit_zero || s->svr.qtype_big) {
+		if(!ssl_printf(ssl, "num.query.type.other"SQ"%u\n", 
+			(unsigned)s->svr.qtype_big)) return 0;
+	}
+	/* CLASS */
+	for(i=0; i<STATS_QCLASS_NUM; i++) {
+		if(inhibit_zero && s->svr.qclass[i] == 0)
+			continue;
+		lt = ldns_lookup_by_id(ldns_rr_classes, i);
+		if(lt && lt->name) {
+			snprintf(nm, sizeof(nm), "%s", lt->name);
+		} else {
+			snprintf(nm, sizeof(nm), "CLASS%d", i);
+		}
+		if(!ssl_printf(ssl, "num.query.class.%s"SQ"%u\n", 
+			nm, (unsigned)s->svr.qclass[i])) return 0;
+	}
+	if(!inhibit_zero || s->svr.qclass_big) {
+		if(!ssl_printf(ssl, "num.query.class.other"SQ"%u\n", 
+			(unsigned)s->svr.qclass_big)) return 0;
+	}
+	/* OPCODE */
+	for(i=0; i<STATS_OPCODE_NUM; i++) {
+		if(inhibit_zero && s->svr.qopcode[i] == 0)
+			continue;
+		lt = ldns_lookup_by_id(ldns_opcodes, i);
+		if(lt && lt->name) {
+			snprintf(nm, sizeof(nm), "%s", lt->name);
+		} else {
+			snprintf(nm, sizeof(nm), "OPCODE%d", i);
+		}
+		if(!ssl_printf(ssl, "num.query.opcode.%s"SQ"%u\n", 
+			nm, (unsigned)s->svr.qopcode[i])) return 0;
+	}
+	/* transport */
+	if(!ssl_printf(ssl, "num.query.tcp"SQ"%u\n", 
+		(unsigned)s->svr.qtcp)) return 0;
+	/* flags */
+	if(!ssl_printf(ssl, "num.query.flags.QR"SQ"%u\n", 
+		(unsigned)s->svr.qbit_QR)) return 0;
+	if(!ssl_printf(ssl, "num.query.flags.AA"SQ"%u\n", 
+		(unsigned)s->svr.qbit_AA)) return 0;
+	if(!ssl_printf(ssl, "num.query.flags.TC"SQ"%u\n", 
+		(unsigned)s->svr.qbit_TC)) return 0;
+	if(!ssl_printf(ssl, "num.query.flags.RD"SQ"%u\n", 
+		(unsigned)s->svr.qbit_RD)) return 0;
+	if(!ssl_printf(ssl, "num.query.flags.RA"SQ"%u\n", 
+		(unsigned)s->svr.qbit_RA)) return 0;
+	if(!ssl_printf(ssl, "num.query.flags.Z"SQ"%u\n", 
+		(unsigned)s->svr.qbit_Z)) return 0;
+	if(!ssl_printf(ssl, "num.query.flags.AD"SQ"%u\n", 
+		(unsigned)s->svr.qbit_AD)) return 0;
+	if(!ssl_printf(ssl, "num.query.flags.CD"SQ"%u\n", 
+		(unsigned)s->svr.qbit_CD)) return 0;
+	if(!ssl_printf(ssl, "num.query.edns.present"SQ"%u\n", 
+		(unsigned)s->svr.qEDNS)) return 0;
+	if(!ssl_printf(ssl, "num.query.edns.DO"SQ"%u\n", 
+		(unsigned)s->svr.qEDNS_DO)) return 0;
+
+	/* RCODE */
+	for(i=0; i<STATS_RCODE_NUM; i++) {
+		if(inhibit_zero && s->svr.ans_rcode[i] == 0)
+			continue;
+		lt = ldns_lookup_by_id(ldns_rcodes, i);
+		if(lt && lt->name) {
+			snprintf(nm, sizeof(nm), "%s", lt->name);
+		} else {
+			snprintf(nm, sizeof(nm), "RCODE%d", i);
+		}
+		if(!ssl_printf(ssl, "num.answer.rcode.%s"SQ"%u\n", 
+			nm, (unsigned)s->svr.qopcode[i])) return 0;
+	}
+	if(!inhibit_zero || s->svr.ans_rcode_nodata) {
+		if(!ssl_printf(ssl, "num.answer.rcode.nodata"SQ"%u\n", 
+			(unsigned)s->svr.ans_rcode_nodata)) return 0;
+	}
+	/* validation */
+	if(!ssl_printf(ssl, "num.answer.secure"SQ"%u\n", 
+		(unsigned)s->svr.ans_secure)) return 0;
+	if(!ssl_printf(ssl, "num.answer.bogus"SQ"%u\n", 
+		(unsigned)s->svr.ans_bogus)) return 0;
+	if(!ssl_printf(ssl, "num.rrset.bogus"SQ"%u\n", 
+		(unsigned)s->svr.rrset_bogus)) return 0;
+	/* threat detection */
+	if(!ssl_printf(ssl, "unwanted.queries"SQ"%u\n", 
+		(unsigned)s->svr.unwanted_queries)) return 0;
+	if(!ssl_printf(ssl, "unwanted.replies"SQ"%u\n", 
+		(unsigned)s->svr.unwanted_replies)) return 0;
+	return 1;
+}
+
 /** do the stats command */
 static void
 do_stats(SSL* ssl, struct daemon_remote* rc)
@@ -584,7 +785,16 @@ do_stats(SSL* ssl, struct daemon_remote* rc)
 	}
 	/* print the thread statistics */
 	total.mesh_time_median /= (double)daemon->num;
-	print_stats(ssl, "total", &total);
+	if(!print_stats(ssl, "total", &total)) 
+		return;
+	if(daemon->cfg->stat_extended) {
+		if(!print_mem(ssl, rc->worker, daemon)) 
+			return;
+		if(!print_uptime(ssl, rc->worker))
+			return;
+		if(!print_ext(ssl, &total))
+			return;
+	}
 }
 
 /** execute a remote control command */

daemon/stats.c

@@ -44,9 +44,12 @@
 #include "daemon/worker.h"
 #include "daemon/daemon.h"
 #include "services/mesh.h"
+#include "services/outside_network.h"
 #include "util/config_file.h"
 #include "util/tube.h"
 #include "util/timehist.h"
+#include "util/net_help.h"
+#include "validator/validator.h"
 
 /** add timers and the values do not overflow or become negative */
 static void
@@ -62,9 +65,10 @@ timeval_add(struct timeval* d, struct timeval* add)
 #endif
 }
 
-void server_stats_init(struct server_stats* stats)
+void server_stats_init(struct server_stats* stats, struct config_file* cfg)
 {
 	memset(stats, 0, sizeof(*stats));
+	stats->extended = cfg->stat_extended;
 }
 
 void server_stats_querymiss(struct server_stats* stats, struct worker* worker)
@@ -92,9 +96,27 @@ void server_stats_log(struct server_stats* stats, struct worker* worker,
 		(unsigned)worker->env.mesh->stats_dropped);
 }
 
+/** get rrsets bogus number from validator */
+static size_t
+get_rrset_bogus(struct worker* worker)
+{
+	int m = modstack_find(&worker->env.mesh->mods, "validator");
+	struct val_env* ve;
+	size_t r;
+	if(m == -1)
+		return 0;
+	ve = (struct val_env*)worker->env.modinfo[m];
+	r = ve->num_rrset_bogus;
+	if(!worker->env.cfg->stat_cumulative)
+		ve->num_rrset_bogus = 0;
+	return r;
+}
+
 void
 server_stats_compile(struct worker* worker, struct stats_info* s)
 {
+	int i;
+
 	s->svr = worker->stats;
 	s->mesh_num_states = worker->env.mesh->all.count;
 	s->mesh_num_reply_states = worker->env.mesh->num_reply_states;
@@ -104,9 +126,23 @@ server_stats_compile(struct worker* worker, struct stats_info* s)
 	s->mesh_replies_sum_wait = worker->env.mesh->replies_sum_wait;
 	s->mesh_time_median = timehist_quartile(worker->env.mesh->histogram,
 		0.50);
+
+	/* add in the values from the mesh */
+	s->svr.ans_secure += worker->env.mesh->ans_secure;
+	s->svr.ans_bogus += worker->env.mesh->ans_bogus;
+	s->svr.ans_rcode_nodata += worker->env.mesh->ans_nodata;
+	for(i=0; i<16; i++)
+		s->svr.ans_rcode[i] += worker->env.mesh->ans_rcode[i];
+	/* values from outside network */
+	s->svr.unwanted_replies = worker->back->unwanted_replies;
+
+	/* get and reset validator rrset bogus number */
+	s->svr.rrset_bogus = get_rrset_bogus(worker);
+
 	if(!worker->env.cfg->stat_cumulative) {
-		server_stats_init(&worker->stats);
+		server_stats_init(&worker->stats, worker->env.cfg);
 		mesh_stats_clear(worker->env.mesh);
+		worker->back->unwanted_replies = 0;
 	}
 }
 
@@ -152,6 +188,37 @@ void server_stats_add(struct stats_info* total, struct stats_info* a)
 	if(a->svr.max_query_list_size > total->svr.max_query_list_size)
 		total->svr.max_query_list_size = a->svr.max_query_list_size;
 
+	if(a->svr.extended) {
+		int i;
+		total->svr.qtype_big += a->svr.qtype_big;
+		total->svr.qclass_big += a->svr.qclass_big;
+		total->svr.qtcp += a->svr.qtcp;
+		total->svr.qbit_QR += a->svr.qbit_QR;
+		total->svr.qbit_AA += a->svr.qbit_AA;
+		total->svr.qbit_TC += a->svr.qbit_TC;
+		total->svr.qbit_RD += a->svr.qbit_RD;
+		total->svr.qbit_RA += a->svr.qbit_RA;
+		total->svr.qbit_Z += a->svr.qbit_Z;
+		total->svr.qbit_AD += a->svr.qbit_AD;
+		total->svr.qbit_CD += a->svr.qbit_CD;
+		total->svr.qEDNS += a->svr.qEDNS;
+		total->svr.qEDNS_DO += a->svr.qEDNS_DO;
+		total->svr.ans_rcode_nodata += a->svr.ans_rcode_nodata;
+		total->svr.ans_secure += a->svr.ans_secure;
+		total->svr.ans_bogus += a->svr.ans_bogus;
+		total->svr.rrset_bogus += a->svr.rrset_bogus;
+		total->svr.unwanted_replies += a->svr.unwanted_replies;
+		total->svr.unwanted_queries += a->svr.unwanted_queries;
+		for(i=0; i<STATS_QTYPE_NUM; i++)
+			total->svr.qtype[i] += a->svr.qtype[i];
+		for(i=0; i<STATS_QCLASS_NUM; i++)
+			total->svr.qclass[i] += a->svr.qclass[i];
+		for(i=0; i<STATS_OPCODE_NUM; i++)
+			total->svr.qopcode[i] += a->svr.qopcode[i];
+		for(i=0; i<STATS_RCODE_NUM; i++)
+			total->svr.ans_rcode[i] += a->svr.ans_rcode[i];
+	}
+
 	total->mesh_num_states += a->mesh_num_states;
 	total->mesh_num_reply_states += a->mesh_num_reply_states;
 	total->mesh_jostled += a->mesh_jostled;
@@ -163,3 +230,49 @@ void server_stats_add(struct stats_info* total, struct stats_info* a)
 	 * added up here, division later*/
 	total->mesh_time_median += a->mesh_time_median;
 }
+
+void server_stats_insquery(struct server_stats* stats, struct comm_point* c,
+	uint16_t qtype, uint16_t qclass, struct edns_data* edns)
+{
+	uint16_t flags = ldns_buffer_read_u16_at(c->buffer, 2);
+	if(qtype < STATS_QTYPE_NUM)
+		stats->qtype[qtype]++;
+	else	stats->qtype_big++;
+	if(qclass < STATS_QCLASS_NUM)
+		stats->qclass[qclass]++;
+	else	stats->qclass_big++;
+	stats->qopcode[ LDNS_OPCODE_WIRE(ldns_buffer_begin(c->buffer)) ]++;
+	if(c->type != comm_udp)
+		stats->qtcp++;
+	if( (flags&BIT_QR) )
+		stats->qbit_QR++;
+	if( (flags&BIT_AA) )
+		stats->qbit_AA++;
+	if( (flags&BIT_TC) )
+		stats->qbit_TC++;
+	if( (flags&BIT_RD) )
+		stats->qbit_RD++;
+	if( (flags&BIT_RA) )
+		stats->qbit_RA++;
+	if( (flags&BIT_Z) )
+		stats->qbit_Z++;
+	if( (flags&BIT_AD) )
+		stats->qbit_AD++;
+	if( (flags&BIT_CD) )
+		stats->qbit_CD++;
+	if(edns->edns_present) {
+		stats->qEDNS++;
+		if( (edns->bits & EDNS_DO) )
+			stats->qEDNS_DO++;
+	}
+}
+
+void server_stats_insrcode(struct server_stats* stats, ldns_buffer* buf)
+{
+	if(stats->extended && ldns_buffer_limit(buf) != 0) {
+		int r = (int)LDNS_RCODE_WIRE( ldns_buffer_begin(buf) );
+		stats->ans_rcode[r] ++;
+		if(r == 0 && LDNS_ANCOUNT( ldns_buffer_begin(buf) ) == 0)
+			stats->ans_rcode_nodata ++;
+	}
+}

daemon/stats.h

@@ -43,6 +43,18 @@
 #ifndef DAEMON_STATS_H
 #define DAEMON_STATS_H
 struct worker;
+struct config_file;
+struct comm_point;
+struct edns_data;
+
+/** number of qtype that is stored for in array */
+#define STATS_QTYPE_NUM 256
+/** number of qclass that is stored for in array */
+#define STATS_QCLASS_NUM 256
+/** number of rcodes in stats */
+#define STATS_RCODE_NUM 16
+/** number of opcodes in stats */
+#define STATS_OPCODE_NUM 16
 
 /** per worker statistics */
 struct server_stats {
@@ -58,6 +70,56 @@ struct server_stats {
 	size_t sum_query_list_size;
 	/** max value of query list size reached. */
 	size_t max_query_list_size;
+
+	/** Extended stats below (bool) */
+	int extended;
+
+	/** qtype stats */
+	size_t qtype[STATS_QTYPE_NUM];
+	/** bigger qtype values not in array */
+	size_t qtype_big;
+	/** qclass stats */
+	size_t qclass[STATS_QCLASS_NUM];
+	/** bigger qclass values not in array */
+	size_t qclass_big;
+	/** query opcodes */
+	size_t qopcode[STATS_OPCODE_NUM];
+	/** number of queries over TCP */
+	size_t qtcp;
+	/** number of queries with QR bit */
+	size_t qbit_QR;
+	/** number of queries with AA bit */
+	size_t qbit_AA;
+	/** number of queries with TC bit */
+	size_t qbit_TC;
+	/** number of queries with RD bit */
+	size_t qbit_RD;
+	/** number of queries with RA bit */
+	size_t qbit_RA;
+	/** number of queries with Z bit */
+	size_t qbit_Z;
+	/** number of queries with AD bit */
+	size_t qbit_AD;
+	/** number of queries with CD bit */
+	size_t qbit_CD;
+	/** number of queries with EDNS OPT record */
+	size_t qEDNS;
+	/** number of queries with EDNS with DO flag */
+	size_t qEDNS_DO;
+	/** answer rcodes */
+	size_t ans_rcode[STATS_RCODE_NUM];
+	/** answers with pseudo rcode 'nodata' */
+	size_t ans_rcode_nodata;
+	/** answers that were secure (AD) */
+	size_t ans_secure;
+	/** answers with bogus content */
+	size_t ans_bogus;
+	/** rrsets marked bogus by validator */
+	size_t rrset_bogus;
+	/** unwanted traffic received on server-facing ports */
+	size_t unwanted_replies;
+	/** unwanted traffic received on client-facing ports */
+	size_t unwanted_queries;
 };
 
 /** 
@@ -87,8 +149,9 @@ struct stats_info {
 /** 
  * Initialize server stats to 0.
  * @param stats: what to init (this is alloced by the caller).
+ * @param cfg: with extended statistics option.
  */
-void server_stats_init(struct server_stats* stats);
+void server_stats_init(struct server_stats* stats, struct config_file* cfg);
 
 /** add query if it missed the cache */
 void server_stats_querymiss(struct server_stats* stats, struct worker* worker);
@@ -127,4 +190,22 @@ void server_stats_reply(struct worker* worker);
  */
 void server_stats_add(struct stats_info* total, struct stats_info* a);
 
+/**
+ * Add stats for this query
+ * @param stats: the stats
+ * @param c: commpoint with type and buffer.
+ * @param qtype: query type
+ * @param qclass: query class
+ * @param edns: edns record
+ */
+void server_stats_insquery(struct server_stats* stats, struct comm_point* c,
+	uint16_t qtype, uint16_t qclass, struct edns_data* edns);
+
+/**
+ * Add rcode for this query.
+ * @param stats: the stats
+ * @param buf: buffer with rcode. If buffer is length0: not counted.
+ */
+void server_stats_insrcode(struct server_stats* stats, ldns_buffer* buf);
+
 #endif /* DAEMON_STATS_H */

daemon/worker.c

@@ -441,6 +441,10 @@ answer_norec_from_cache(struct worker* worker, struct query_info* qinfo,
 			error_encode(repinfo->c->buffer, LDNS_RCODE_SERVFAIL, 
 				&msg->qinfo, id, flags, edns);
 			regional_free_all(worker->scratchpad);
+			if(worker->stats.extended) {
+				worker->stats.ans_bogus++;
+				worker->stats.ans_rcode[LDNS_RCODE_SERVFAIL]++;
+			}
 			return 1;
 		case sec_status_secure:
 			/* all rrsets are secure */
@@ -470,6 +474,10 @@ answer_norec_from_cache(struct worker* worker, struct query_info* qinfo,
 			&msg->qinfo, id, flags, edns);
 	}
 	regional_free_all(worker->scratchpad);
+	if(worker->stats.extended) {
+		if(secure) worker->stats.ans_secure++;
+		server_stats_insrcode(&worker->stats, repinfo->c->buffer);
+	}
 	return 1;
 }
 
@@ -557,6 +565,10 @@ answer_from_cache(struct worker* worker, struct query_info* qinfo,
 		rrset_array_unlock_touch(worker->env.rrset_cache, 
 			worker->scratchpad, rep->ref, rep->rrset_count);
 		regional_free_all(worker->scratchpad);
+		if(worker->stats.extended) {
+			worker->stats.ans_bogus ++;
+			worker->stats.ans_rcode[LDNS_RCODE_SERVFAIL] ++;
+		}
 		return 1;
 	} else if( rep->security == sec_status_unchecked && must_validate) {
 		verbose(VERB_ALGO, "Cache reply: unchecked entry needs "
@@ -590,6 +602,10 @@ answer_from_cache(struct worker* worker, struct query_info* qinfo,
 	rrset_array_unlock_touch(worker->env.rrset_cache, worker->scratchpad,
 		rep->ref, rep->rrset_count);
 	regional_free_all(worker->scratchpad);
+	if(worker->stats.extended) {
+		if(secure) worker->stats.ans_secure++;
+		server_stats_insrcode(&worker->stats, repinfo->c->buffer);
+	}
 	/* go and return this buffer to the client */
 	return 1;
 }
@@ -704,6 +720,8 @@ worker_handle_request(struct comm_point* c, void* arg, int error,
 		repinfo->addrlen);
 	if(acl == acl_deny) {
 		comm_point_drop_reply(repinfo);
+		if(worker->stats.extended)
+			worker->stats.unwanted_queries++;
 		return 0;
 	} else if(acl == acl_refuse) {
 		ldns_buffer_set_limit(c->buffer, LDNS_HEADER_SIZE);
@@ -715,6 +733,8 @@ worker_handle_request(struct comm_point* c, void* arg, int error,
 		log_addr(VERB_ALGO, "refused query from",
 			&repinfo->addr, repinfo->addrlen);
 		log_buf(VERB_ALGO, "refuse", c->buffer);
+		if(worker->stats.extended)
+			worker->stats.unwanted_queries++;
 		return 1;
 	}
 	if((ret=worker_check_request(c->buffer, worker)) != 0) {
@@ -734,6 +754,7 @@ worker_handle_request(struct comm_point* c, void* arg, int error,
 		LDNS_QR_SET(ldns_buffer_begin(c->buffer));
 		LDNS_RCODE_SET(ldns_buffer_begin(c->buffer), 
 			LDNS_RCODE_FORMERR);
+		server_stats_insrcode(&worker->stats, c->buffer);
 		return 1;
 	}
 	if(qinfo.qtype == LDNS_RR_TYPE_AXFR || 
@@ -742,12 +763,17 @@ worker_handle_request(struct comm_point* c, void* arg, int error,
 		LDNS_QR_SET(ldns_buffer_begin(c->buffer));
 		LDNS_RCODE_SET(ldns_buffer_begin(c->buffer), 
 			LDNS_RCODE_REFUSED);
+		if(worker->stats.extended) {
+			worker->stats.qtype[qinfo.qtype]++;
+			server_stats_insrcode(&worker->stats, c->buffer);
+		}
 		return 1;
 	}
 	if((ret=parse_edns_from_pkt(c->buffer, &edns)) != 0) {
 		verbose(VERB_ALGO, "worker parse edns: formerror.");
 		LDNS_QR_SET(ldns_buffer_begin(c->buffer));
 		LDNS_RCODE_SET(ldns_buffer_begin(c->buffer), ret);
+		server_stats_insrcode(&worker->stats, c->buffer);
 		return 1;
 	}
 	if(edns.edns_present && edns.edns_version != 0) {
@@ -780,14 +806,19 @@ worker_handle_request(struct comm_point* c, void* arg, int error,
 		ldns_buffer_flip(c->buffer);
 		return 1;
 	}
+	if(worker->stats.extended)
+		server_stats_insquery(&worker->stats, c, qinfo.qtype,
+			qinfo.qclass, &edns);
 	if(c->type != comm_udp)
 		edns.udp_size = 65535; /* max size for TCP replies */
 	if(qinfo.qclass == LDNS_RR_CLASS_CH && answer_chaos(worker, &qinfo,
 		&edns, c->buffer)) {
+		server_stats_insrcode(&worker->stats, c->buffer);
 		return 1;
 	}
 	if(local_zones_answer(worker->daemon->local_zones, &qinfo, &edns, 
 		c->buffer, worker->scratchpad)) {
+		server_stats_insrcode(&worker->stats, c->buffer);
 		return (ldns_buffer_limit(c->buffer) != 0);
 	}
 	if(!(LDNS_RD_WIRE(ldns_buffer_begin(c->buffer))) &&
@@ -799,6 +830,7 @@ worker_handle_request(struct comm_point* c, void* arg, int error,
 		LDNS_RCODE_SET(ldns_buffer_begin(c->buffer), 
 			LDNS_RCODE_REFUSED);
 		ldns_buffer_flip(c->buffer);
+		server_stats_insrcode(&worker->stats, c->buffer);
 		log_addr(VERB_ALGO, "refused nonrec (cache snoop) query from",
 			&repinfo->addr, repinfo->addrlen);
 		return 1;
@@ -903,8 +935,9 @@ void worker_stat_timer_cb(void* arg)
 	mesh_stats(worker->env.mesh, "mesh has");
 	worker_mem_report(worker, NULL);
 	if(!worker->daemon->cfg->stat_cumulative) {
-		server_stats_init(&worker->stats);
+		server_stats_init(&worker->stats, worker->env.cfg);
 		mesh_stats_clear(worker->env.mesh);
+		worker->back->unwanted_replies = 0;
 	}
 	/* start next timer */
 	worker_restart_timer(worker);
@@ -1036,7 +1069,7 @@ worker_init(struct worker* worker, struct config_file *cfg,
 	}
 	worker->request_size = cfg->num_queries_per_thread;
 
-	server_stats_init(&worker->stats);
+	server_stats_init(&worker->stats, cfg);
 	alloc_init(&worker->alloc, &worker->daemon->superalloc, 
 		worker->thread_num);
 	alloc_set_id_cleanup(&worker->alloc, &worker_alloc_cleanup, worker);

doc/Changelog

@@ -1,5 +1,7 @@
 16 September 2008: Wouter
 	- extended-statistics: yesno config option.
+	- unwanted replies spoof nearmiss detector.
+	- iana portlist updated.
 
 15 September 2008: Wouter
 	- working start, stop, reload commands for unbound-control.

services/mesh.c

@@ -599,6 +599,8 @@ mesh_send_reply(struct mesh_state* m, int rcode, struct reply_info* rep,
 	if(m->s.env->need_to_validate && !(r->qflags&BIT_CD) && rep && 
 		rep->security <= sec_status_bogus) {
 		rcode = LDNS_RCODE_SERVFAIL;
+		if(m->s.env->cfg->stat_extended) 
+			m->s.env->mesh->ans_bogus++;
 	}
 	if(rep && rep->security == sec_status_secure)
 		secure = 1;
@@ -651,6 +653,15 @@ mesh_send_reply(struct mesh_state* m, int rcode, struct reply_info* rep,
 	m->s.env->mesh->replies_sent++;
 	timeval_add(&m->s.env->mesh->replies_sum_wait, &duration);
 	timehist_insert(m->s.env->mesh->histogram, &duration);
+	if(m->s.env->cfg->stat_extended) {
+		uint16_t rc = FLAGS_GET_RCODE(ldns_buffer_read_u16_at(r->
+			query_reply.c->buffer, 2));
+		if(secure) m->s.env->mesh->ans_secure++;
+		m->s.env->mesh->ans_rcode[ rc ] ++;
+		if(rc == 0 && LDNS_ANCOUNT(ldns_buffer_begin(r->
+			query_reply.c->buffer)) == 0)
+			m->s.env->mesh->ans_nodata++;
+	}
 }
 
 void mesh_query_done(struct mesh_state* mstate)
@@ -891,6 +902,10 @@ mesh_stats_clear(struct mesh_area* mesh)
 	mesh->stats_jostled = 0;
 	mesh->stats_dropped = 0;
 	timehist_clear(mesh->histogram);
+	mesh->ans_secure = 0;
+	mesh->ans_bogus = 0;
+	memset(&mesh->ans_rcode[0], 0, sizeof(size_t)*16);
+	mesh->ans_nodata = 0;
 }
 
 size_t 

services/mesh.h

@@ -113,6 +113,14 @@ struct mesh_area {
 	struct timeval replies_sum_wait;
 	/** histogram of time values */
 	struct timehist* histogram;
+	/** (extended stats) secure replies */
+	size_t ans_secure;
+	/** (extended stats) bogus replies */
+	size_t ans_bogus;
+	/** (extended stats) rcodes in replies */
+	size_t ans_rcode[16];
+	/** (extended stats) rcode nodata in replies */
+	size_t ans_nodata;
 
 	/** double linked list of the run-to-completion query states.
 	 * These are query states with a reply */

services/modstack.c

@@ -165,3 +165,14 @@ modstack_desetup(struct module_stack* stack, struct module_env* env)
         free(stack->mod);
         stack->mod = NULL;
 }
+
+int 
+modstack_find(struct module_stack* stack, const char* name)
+{
+	int i;
+        for(i=0; i<stack->num; i++) {
+		if(strcmp(stack->mod[i]->name, name) == 0)
+			return i;
+	}
+	return -1;
+}

services/modstack.h

@@ -96,4 +96,12 @@ int modstack_setup(struct module_stack* stack, const char* module_conf,
  */
 void modstack_desetup(struct module_stack* stack, struct module_env* env);
 
+/**
+ * Find index of module by name.
+ * @param stack: to look in
+ * @param name: the name to look for
+ * @return -1 on failure, otherwise index number.
+ */
+int modstack_find(struct module_stack* stack, const char* name);
+
 #endif /* SERVICES_MODSTACK_H */

services/outside_network.c

@@ -329,6 +329,7 @@ outnet_udp_cb(struct comm_point* c, void* arg, int error,
 	if(!p) {
 		verbose(VERB_QUERY, "received unwanted or unsolicited udp reply dropped.");
 		log_buf(VERB_ALGO, "dropped message", c->buffer);
+		outnet->unwanted_replies++;
 		return 0;
 	}
 
@@ -337,6 +338,7 @@ outnet_udp_cb(struct comm_point* c, void* arg, int error,
 	if(p->pc->cp != c) {
 		verbose(VERB_QUERY, "received reply id,addr on wrong port. "
 			"dropped.");
+		outnet->unwanted_replies++;
 		return 0;
 	}
 	comm_timer_disable(p->timer);

services/outside_network.h

@@ -76,6 +76,8 @@ struct outside_network {
 	size_t svcd_overhead;
 	/** use x20 bits to encode additional ID random bits */
 	int use_caps_for_id;
+	/** number of unwanted replies received */
+	size_t unwanted_replies;
 
 	/** linked list of available commpoints, unused file descriptors,
 	 * for use as outgoing UDP ports. cp.fd=-1 in them. */

util/iana_ports.inc

@@ -3777,6 +3777,7 @@
 4183,
 4184,
 4185,
+4188,
 4199,
 4300,
 4301,

util/net_help.h

@@ -54,6 +54,8 @@
 #define BIT_CD 0x0010
 /** AD flag */
 #define BIT_AD 0x0020
+/** Z flag */
+#define BIT_Z  0x0040
 /** RA flag */
 #define BIT_RA 0x0080
 /** RD flag */

validator/val_utils.c

@@ -341,6 +341,7 @@ val_verify_rrset(struct module_env* env, struct val_env* ve,
 			d->ttl = ve->bogus_ttl;
 			/* leave RR specific TTL: not used for determine
 			 * if RRset timed out and clients see proper value. */
+			ve->num_rrset_bogus++;
 		}
 		/* if status updated - store in cache for reuse */
 		rrset_update_sec_status(env->rrset_cache, rrset, *env->now);

validator/validator.h

@@ -107,6 +107,9 @@ struct val_env {
 	 * in the keysize array.
 	 */
 	size_t* nsec3_maxiter;
+
+	/** number of times rrsets marked bogus */
+	size_t num_rrset_bogus;
 };
 
 /**