Serviced queries in outside network service get full EDNS, UDP retry and

TCP fallback attention. git-svn-id: file:///svn/unbound/trunk@326 be551aaa-1e26-0410-a405-d3ace91eadb9
2026-02-09 22:03:15 -05:00 · 2007-05-21 15:10:55 +00:00 · 2007-05-21 15:10:55 +00:00 · 1a90ff7b67
commit 1a90ff7b67
parent 69bfd93616
9 changed files with 608 additions and 72 deletions
--- a/daemon/worker.c
+++ b/daemon/worker.c
@ -317,6 +317,8 @@ answer_from_cache(struct worker* worker, struct lruhash_entry* e, uint16_t id,
 		replyerror_fillbuf(LDNS_RCODE_SERVFAIL, repinfo, id,
 			flags, &mrentry->key);
 	}
+	/* cannot send the reply right now, because blocking network syscall
+	 * is bad while holding locks. */
 	/* unlock */
 	for(i=0; i<rep->rrset_count; i++) {
 		if(i>0 && rep->ref[i].key == rep->ref[i-1].key)
@ -324,6 +326,7 @@ answer_from_cache(struct worker* worker, struct lruhash_entry* e, uint16_t id,
 		h[i] = rep->ref[i].key->entry.hash;
 		lock_rw_unlock(&rep->ref[i].key->entry.lock);
 	}
+	/* still holding msgreply lock to touch LRU, so cannot send reply yet*/
 	/* LRU touch, with no rrset locks held */
 	for(i=0; i<rep->rrset_count; i++) {
 		if(i>0 && rep->ref[i].key == rep->ref[i-1].key)
@ -586,6 +589,20 @@ worker_init(struct worker* worker, struct config_file *cfg,
 	} else { /* !do_sigs */
 		worker->comsig = 0;
 	}
+	/* init random(), large table size. */
+	if(!(worker->rndstate = (struct ub_randstate*)calloc(1,
+		sizeof(struct ub_randstate)))) {
+		log_err("malloc rndtable failed.");
+		worker_delete(worker);
+		return 0;
+	}
+	seed = (unsigned int)time(NULL) ^ (unsigned int)getpid() ^
+		(unsigned int)worker->thread_num;
+	if(!ub_initstate(seed, worker->rndstate, RND_STATE_SIZE)) {
+		log_err("could not init random numbers.");
+		worker_delete(worker);
+		return 0;
+	}
 	worker->front = listen_create(worker->base, ports,
 		buffer_size, worker_handle_request, worker);
 	if(!worker->front) {
@ -598,26 +615,13 @@ worker_init(struct worker* worker, struct config_file *cfg,
 	worker->back = outside_network_create(worker->base,
 		buffer_size, (size_t)cfg->outgoing_num_ports, cfg->ifs, 
 		cfg->num_ifs, cfg->do_ip4, cfg->do_ip6, startport, 
-		cfg->do_tcp?cfg->outgoing_num_tcp:0);
+		cfg->do_tcp?cfg->outgoing_num_tcp:0, 
+		worker->daemon->env->infra_cache, worker->rndstate);
 	if(!worker->back) {
 		log_err("could not create outgoing sockets");
 		worker_delete(worker);
 		return 0;
 	}
-	/* init random(), large table size. */
-	if(!(worker->rndstate = (struct ub_randstate*)calloc(1,
-		sizeof(struct ub_randstate)))) {
-		log_err("malloc rndtable failed.");
-		worker_delete(worker);
-		return 0;
-	}
-	seed = (unsigned int)time(NULL) ^ (unsigned int)getpid() ^
-		(unsigned int)worker->thread_num;
-	if(!ub_initstate(seed, worker->rndstate, RND_STATE_SIZE)) {
-		log_err("could not init random numbers.");
-		worker_delete(worker);
-		return 0;
-	}
 	if(worker->thread_num != 0) {
 		/* start listening to commands */
 		if(!(worker->cmd_com=comm_point_create_local(worker->base, 
@ -691,8 +695,8 @@ worker_send_query(ldns_buffer* pkt, struct sockaddr_storage* addr,
 	if(use_tcp) {
 		return pending_tcp_query(worker->back, pkt, addr, addrlen, 
 			timeout, worker_handle_reply, q->work_info, 
-			worker->rndstate);
+			worker->rndstate) != 0;
 	}
 	return pending_udp_query(worker->back, pkt, addr, addrlen, timeout,
-		worker_handle_reply, q->work_info, worker->rndstate);
+		worker_handle_reply, q->work_info, worker->rndstate) != 0;
 }
--- a/doc/Changelog
+++ b/doc/Changelog
@ -1,3 +1,8 @@
+21 May 2007: Wouter
+	- small comment on hash table locking.
+	- outside network serviced queries, contain edns and tcp fallback,
+	  and udp retries and rtt timing.
+
 16 May 2007: Wouter
 	- lruhash_touch() would cause locking order problems. Fixup in 
 	  lock-verify in case locking cycle is found.
--- a/doc/TODO
+++ b/doc/TODO
@ -21,3 +21,4 @@ o detect OS/400 pthreads implementation that allows upgrading to writelock
 o understand synthesized DNAMEs, so those TTL=0 packets are cached properly.
 o understand NSEC/NSEC3, aggressive negative caching, so that updates to
  NSEC/NSEC3 will result in proper negative responses.
+o fallback without EDNS if result is NOTIMPL, now only on FORMERR like in java.
--- a/services/outside_network.c
+++ b/services/outside_network.c
@ -42,6 +42,9 @@

 #include "services/outside_network.h"
 #include "services/listen_dnsport.h"
+#include "services/cache/infra.h"
+#include "util/data/msgparse.h"
+#include "util/data/msgreply.h"
 #include "util/netevent.h"
 #include "util/log.h"
 #include "util/net_help.h"
@ -59,27 +62,30 @@
 #define INET_SIZE 4
 /** byte size of ip6 address */
 #define INET6_SIZE 16 
+/** number of retries on outgoing UDP queries */
+#define OUTBOUND_UDP_RETRY 4

-/** compare function of pending rbtree */
-static int 
-pending_cmp(const void* key1, const void* key2)
+/** callback for serviced query UDP answers. */
+static int serviced_udp_callback(struct comm_point* c, void* arg, int error,
+        struct comm_reply* rep);
+/** initiate TCP transaction for serviced query */
+static void serviced_tcp_initiate(struct outside_network* outnet, 
+	struct serviced_query* sq, ldns_buffer* buff);
+
+/** compare sockaddr */
+static int
+sockaddr_cmp(struct sockaddr_storage* addr1, socklen_t len1, 
+	struct sockaddr_storage* addr2, socklen_t len2)
 {
-	struct pending *p1 = (struct pending*)key1;
-	struct pending *p2 = (struct pending*)key2;
-	struct sockaddr_in* p1_in = (struct sockaddr_in*)&p1->addr;
-	struct sockaddr_in* p2_in = (struct sockaddr_in*)&p2->addr;
-	struct sockaddr_in6* p1_in6 = (struct sockaddr_in6*)&p1->addr;
-	struct sockaddr_in6* p2_in6 = (struct sockaddr_in6*)&p2->addr;
-	if(p1->id < p2->id)
+	struct sockaddr_in* p1_in = (struct sockaddr_in*)addr1;
+	struct sockaddr_in* p2_in = (struct sockaddr_in*)addr2;
+	struct sockaddr_in6* p1_in6 = (struct sockaddr_in6*)addr1;
+	struct sockaddr_in6* p2_in6 = (struct sockaddr_in6*)addr2;
+	if(len1 < len2)
 		return -1;
-	if(p1->id > p2->id)
+	if(len1 > len2)
 		return 1;
-	log_assert(p1->id == p2->id);
-	if(p1->addrlen < p2->addrlen)
-		return -1;
-	if(p1->addrlen > p2->addrlen)
-		return 1;
-	log_assert(p1->addrlen == p2->addrlen);
+	log_assert(len1 == len2);
 	if( p1_in->sin_family < p2_in->sin_family)
 		return -1;
 	if( p1_in->sin_family > p2_in->sin_family)
@ -105,10 +111,46 @@ pending_cmp(const void* key1, const void* key2)
 			INET6_SIZE);
 	} else {
 		/* eek unknown type, perform this comparison for sanity. */
-		return memcmp(&p1->addr, &p2->addr, p1->addrlen);
+		return memcmp(addr1, addr2, len1);
 	}
 }

+/** compare function of pending rbtree */
+static int 
+pending_cmp(const void* key1, const void* key2)
+{
+	struct pending *p1 = (struct pending*)key1;
+	struct pending *p2 = (struct pending*)key2;
+	if(p1->id < p2->id)
+		return -1;
+	if(p1->id > p2->id)
+		return 1;
+	log_assert(p1->id == p2->id);
+	return sockaddr_cmp(&p1->addr, p1->addrlen, &p2->addr, p2->addrlen);
+}
+
+/** compare function of serviced query rbtree */
+static int 
+serviced_cmp(const void* key1, const void* key2)
+{
+	struct serviced_query* q1 = (struct serviced_query*)key1;
+	struct serviced_query* q2 = (struct serviced_query*)key2;
+	int r;
+	if(q1->qbuflen < q2->qbuflen)
+		return -1;
+	if(q1->qbuflen > q2->qbuflen)
+		return 1;
+	log_assert(q1->qbuflen == q2->qbuflen);
+	if((r = memcmp(q1->qbuf, q2->qbuf, q1->qbuflen)) != 0)
+		return r;
+	if(q1->dnssec != q2->dnssec) {
+		if(q1->dnssec < q2->dnssec)
+			return -1;
+		return 1;
+	}
+	return sockaddr_cmp(&q1->addr, q1->addrlen, &q2->addr, q2->addrlen);
+}
+
 /** delete waiting_tcp entry. Does not unlink from waiting list. 
 * @param w: to delete.
 */
@ -182,6 +224,19 @@ use_free_buffer(struct outside_network* outnet)
 	}
 }

+/** decomission a tcp buffer, closes commpoint and frees waiting_tcp entry */
+static void
+decomission_pending_tcp(struct outside_network* outnet, 
+	struct pending_tcp* pend)
+{
+	comm_point_close(pend->c);
+	pend->next_free = outnet->tcp_free;
+	outnet->tcp_free = pend;
+	waiting_tcp_delete(pend->query);
+	pend->query = NULL;
+	use_free_buffer(outnet);
+}
+
 /** callback for pending tcp connections */
 static int 
 outnet_tcp_cb(struct comm_point* c, void* arg, int error,
@ -203,12 +258,7 @@ outnet_tcp_cb(struct comm_point* c, void* arg, int error,
 		}
 	}
 	(void)(*pend->query->cb)(c, pend->query->cb_arg, error, reply_info);
-	comm_point_close(c);
-	pend->next_free = outnet->tcp_free;
-	outnet->tcp_free = pend;
-	waiting_tcp_delete(pend->query);
-	pend->query = NULL;
-	use_free_buffer(outnet);
+	decomission_pending_tcp(outnet, pend);
 	return 0;
 }

@ -255,8 +305,10 @@ outnet_udp_cb(struct comm_point* c, void* arg, int error,
 	}
 	comm_timer_disable(p->timer);
 	verbose(VERB_ALGO, "outnet handle udp reply");
+	/* delete from tree first in case callback creates a retry */
+	(void)rbtree_delete(outnet->pending, p->node.key);
 	(void)(*p->cb)(p->c, p->cb_arg, NETEVENT_NOERROR, reply_info);
-	pending_delete(outnet, p);
+	pending_delete(NULL, p);
 	return 0;
 }

@ -402,7 +454,8 @@ create_pending_tcp(struct outside_network* outnet, size_t bufsize)
 struct outside_network* 
 outside_network_create(struct comm_base *base, size_t bufsize, 
 	size_t num_ports, char** ifs, int num_ifs, int do_ip4, 
-	int do_ip6, int port_base, size_t num_tcp)
+	int do_ip6, int port_base, size_t num_tcp, struct infra_cache* infra,
+	struct ub_randstate* rnd)
 {
 	struct outside_network* outnet = (struct outside_network*)
 		calloc(1, sizeof(struct outside_network));
@ -413,6 +466,8 @@ outside_network_create(struct comm_base *base, size_t bufsize,
 	}
 	outnet->base = base;
 	outnet->num_tcp = num_tcp;
+	outnet->infra = infra;
+	outnet->rnd = rnd;
 #ifndef INET6
 	do_ip6 = 0;
 #endif
@ -425,6 +480,7 @@ outside_network_create(struct comm_base *base, size_t bufsize,
 		!(outnet->udp6_ports = (struct comm_point **)calloc(
 			outnet->num_udp6+1, sizeof(struct comm_point*))) ||
 		!(outnet->pending = rbtree_create(pending_cmp)) ||
+		!(outnet->serviced = rbtree_create(serviced_cmp)) ||
 		!create_pending_tcp(outnet, bufsize)) {
 		log_err("malloc failed");
 		outside_network_delete(outnet);
@ -481,6 +537,21 @@ pending_node_del(rbnode_t* node, void* arg)
 	pending_delete(outnet, pend);
 }

+/** helper serviced delete */
+static void
+serviced_node_del(rbnode_t* node, void* ATTR_UNUSED(arg))
+{
+	struct serviced_query* sq = (struct serviced_query*)node;
+	struct service_callback* p = sq->cblist, *np;
+	free(sq->qbuf);
+	while(p) {
+		np = p->next;
+		free(p);
+		p = np;
+	}
+	free(sq);
+}
+
 void 
 outside_network_delete(struct outside_network* outnet)
 {
@ -492,6 +563,10 @@ outside_network_delete(struct outside_network* outnet)
 		traverse_postorder(outnet->pending, pending_node_del, NULL);
 		free(outnet->pending);
 	}
+	if(outnet->serviced) {
+		traverse_postorder(outnet->serviced, serviced_node_del, NULL);
+		free(outnet->serviced);
+	}
 	if(outnet->udp_buff)
 		ldns_buffer_free(outnet->udp_buff);
 	if(outnet->udp4_ports) {
@ -648,7 +723,7 @@ select_port(struct outside_network* outnet, struct pending* pend,
 }


-int 
+struct pending* 
 pending_udp_query(struct outside_network* outnet, ldns_buffer* packet, 
 	struct sockaddr_storage* addr, socklen_t addrlen, int timeout,
 	comm_point_callback_t* cb, void* cb_arg, struct ub_randstate* rnd)
@ -659,7 +734,7 @@ pending_udp_query(struct outside_network* outnet, ldns_buffer* packet,
 	/* create pending struct and change ID to be unique */
 	if(!(pend=new_pending(outnet, packet, addr, addrlen, cb, cb_arg, 
 		rnd))) {
-		return 0;
+		return NULL;
 	}
 	select_port(outnet, pend, rnd);

@ -667,7 +742,7 @@ pending_udp_query(struct outside_network* outnet, ldns_buffer* packet,
 	if(!comm_point_send_udp_msg(pend->c, packet, (struct sockaddr*)addr, 
 		addrlen)) {
 		pending_delete(outnet, pend);
-		return 0;
+		return NULL;
 	}

 	/* system calls to set timeout after sending UDP to make roundtrip
@ -675,7 +750,7 @@ pending_udp_query(struct outside_network* outnet, ldns_buffer* packet,
 	tv.tv_sec = timeout;
 	tv.tv_usec = 0;
 	comm_timer_set(pend->timer, &tv);
-	return 1;
+	return pend;
 }

 /** callback for outgoing TCP timer event */
@ -710,7 +785,7 @@ outnet_tcptimer(void* arg)
 	use_free_buffer(outnet);
 }

-int 
+struct waiting_tcp* 
 pending_tcp_query(struct outside_network* outnet, ldns_buffer* packet, 
 	struct sockaddr_storage* addr, socklen_t addrlen, int timeout,
 	comm_point_callback_t* callback, void* callback_arg,
@ -724,11 +799,11 @@ pending_tcp_query(struct outside_network* outnet, ldns_buffer* packet,
 	w = (struct waiting_tcp*)malloc(sizeof(struct waiting_tcp) 
 		+ (pend?0:ldns_buffer_limit(packet)));
 	if(!w) {
-		return 0;
+		return NULL;
 	}
 	if(!(w->timer = comm_timer_create(outnet->base, outnet_tcptimer, w))) {
 		free(w);
-		return 0;
+		return NULL;
 	}
 	w->pkt = NULL;
 	w->pkt_len = ldns_buffer_limit(packet);
@ -747,7 +822,7 @@ pending_tcp_query(struct outside_network* outnet, ldns_buffer* packet,
 		/* we have a buffer available right now */
 		if(!outnet_tcp_take_into_use(w, ldns_buffer_begin(packet))) {
 			waiting_tcp_delete(w);
-			return 0;
+			return NULL;
 		}
 	} else {
 		/* queue up */
@ -759,5 +834,348 @@ pending_tcp_query(struct outside_network* outnet, ldns_buffer* packet,
 		else	outnet->tcp_wait_first = w;
 		outnet->tcp_wait_last = w;
 	}
+	return w;
+}
+
+/** create query for serviced queries. */
+static void
+serviced_gen_query(ldns_buffer* buff, uint8_t* qname, size_t qnamelen, 
+	uint16_t qtype, uint16_t qclass, uint16_t flags)
+{
+	ldns_buffer_clear(buff);
+	/* skip id */
+	ldns_buffer_write_u16(buff, flags);
+	ldns_buffer_write_u16(buff, 1); /* qdcount */
+	ldns_buffer_write_u16(buff, 0); /* ancount */
+	ldns_buffer_write_u16(buff, 0); /* nscount */
+	ldns_buffer_write_u16(buff, 0); /* arcount */
+	ldns_buffer_write(buff, qname, qnamelen);
+	ldns_buffer_write_u16(buff, qtype);
+	ldns_buffer_write_u16(buff, qclass);
+	ldns_buffer_flip(buff);
+}
+
+/** lookup serviced query in serviced query rbtree */
+static struct serviced_query*
+lookup_serviced(struct outside_network* outnet, ldns_buffer* buff, int dnssec,
+	struct sockaddr_storage* addr, socklen_t addrlen)
+{
+	struct serviced_query key;
+	key.node.key = &key;
+	key.qbuf = ldns_buffer_begin(buff);
+	key.qbuflen = ldns_buffer_limit(buff);
+	key.dnssec = dnssec;
+	memcpy(&key.addr, addr, addrlen);
+	key.addrlen = addrlen;
+	key.outnet = outnet;
+	return (struct serviced_query*)rbtree_search(outnet->serviced, &key);
+}
+
+/** Create new serviced entry */
+static struct serviced_query*
+serviced_create(struct outside_network* outnet, ldns_buffer* buff, int dnssec,
+        struct sockaddr_storage* addr, socklen_t addrlen)
+{
+	struct serviced_query* sq = (struct serviced_query*)malloc(sizeof(*sq));
+	rbnode_t* ins;
+	if(!sq) 
+		return NULL;
+	sq->node.key = sq;
+	sq->qbuf = memdup(ldns_buffer_begin(buff), ldns_buffer_limit(buff));
+	if(!sq->qbuf) {
+		free(sq);
+		return NULL;
+	}
+	sq->qbuflen = ldns_buffer_limit(buff);
+	sq->dnssec = dnssec;
+	memcpy(&sq->addr, addr, addrlen);
+	sq->addrlen = addrlen;
+	sq->outnet = outnet;
+	sq->cblist = NULL;
+	sq->pending = NULL;
+	sq->status = serviced_initial;
+	sq->retry = 0;
+	ins = rbtree_insert(outnet->serviced, &sq->node);
+	log_assert(ins != NULL); /* must not be already present */
+	return sq;
+}
+
+/** remove waiting tcp from the outnet waiting list */
+static void
+waiting_list_remove(struct outside_network* outnet, struct waiting_tcp* w)
+{
+	struct waiting_tcp* p = outnet->tcp_wait_first, *prev = NULL;
+	while(p) {
+		if(p == w) {
+			/* remove w */
+			if(prev)
+				prev->next_waiting = w->next_waiting;
+			else	outnet->tcp_wait_first = w->next_waiting;
+			if(outnet->tcp_wait_last == w)
+				outnet->tcp_wait_last = prev;
+			return;
+		}
+		prev = p;
+		p = p->next_waiting;
+	}
+}
+
+/** cleanup serviced query entry */
+static void
+serviced_delete(struct serviced_query* sq)
+{
+	if(sq->pending) {
+		/* clear up the pending query */
+		if(sq->status == serviced_query_UDP_EDNS ||
+			sq->status == serviced_query_UDP) {
+			struct pending* p = (struct pending*)sq->pending;
+			pending_delete(sq->outnet, p);
+		} else {
+			struct waiting_tcp* p = (struct waiting_tcp*)
+				sq->pending;
+			if(p->pkt == NULL) {
+				decomission_pending_tcp(sq->outnet, 
+					(struct pending_tcp*)p->next_waiting);
+			} else {
+				waiting_list_remove(sq->outnet, p);
+			}
+		}
+	}
+	/* does not delete from tree, caller has to do that */
+	serviced_node_del(&sq->node, NULL);
+}
+
+/** put serviced query into a buffer */
+static void
+serviced_encode(struct serviced_query* sq, ldns_buffer* buff, int with_edns)
+{
+	/* generate query */
+	ldns_buffer_clear(buff);
+	ldns_buffer_write_u16(buff, 0); /* id placeholder */
+	ldns_buffer_write(buff, sq->qbuf, sq->qbuflen);
+	if(with_edns) {
+		/* add edns section */
+		struct edns_data edns;
+		edns.edns_present = 1;
+		edns.ext_rcode = 0;
+		edns.edns_version = EDNS_ADVERTISED_VERSION;
+		edns.udp_size = EDNS_ADVERTISED_SIZE;
+		edns.bits = 0;
+		if(sq->dnssec)
+			edns.bits = EDNS_DO;
+		attach_edns_record(buff, &edns);
+	}
+	ldns_buffer_flip(buff);
+}
+
+/**
+ * Perform serviced query UDP sending operation.
+ * Sends UDP with EDNS, unless infra host marked non EDNS.
+ * @param sq: query to send.
+ * @param buff: buffer scratch space.
+ * @return 0 on error.
+ */
+static int
+serviced_udp_send(struct serviced_query* sq, ldns_buffer* buff)
+{
+	int rtt, vs;
+	time_t now = time(0);
+
+	if(!infra_host(sq->outnet->infra, &sq->addr, sq->addrlen, now, &vs,
+		&rtt))
+		return 0;
+	if(sq->status == serviced_initial) {
+		if(vs != -1)
+			sq->status = serviced_query_UDP_EDNS;
+		else 	sq->status = serviced_query_UDP;
+	}
+	serviced_encode(sq, buff, sq->status == serviced_query_UDP_EDNS);
+	sq->last_sent_time = now;
+	/* round rtt to whole seconds for now. TODO better timing */
+	rtt = rtt/1000 + 1;
+	sq->pending = pending_udp_query(sq->outnet, buff, &sq->addr, 
+		sq->addrlen, rtt, serviced_udp_callback, sq, sq->outnet->rnd);
+	if(!sq->pending)
+		return 0;
 	return 1;
 }
+
+/** call the callbacks for a serviced query */
+static void
+serviced_callbacks(struct serviced_query* sq, int error, struct comm_point* c)
+{
+	struct service_callback* p = sq->cblist;
+	while(p) {
+		(void)(*p->cb)(c, p->cb_arg, error, NULL);
+		p = p->next;
+	}
+}
+
+/** TCP reply or error callback for serviced queries */
+static int 
+serviced_tcp_callback(struct comm_point* c, void* arg, int error,
+        struct comm_reply* ATTR_UNUSED(rep))
+{
+	struct serviced_query* sq = (struct serviced_query*)arg;
+	sq->pending = NULL; /* removed after this callback */
+	if(error==NETEVENT_NOERROR && LDNS_RCODE_WIRE(ldns_buffer_begin(
+		c->buffer)) == LDNS_RCODE_FORMERR && 
+		sq->status == serviced_query_TCP_EDNS) {
+		if(!infra_edns_update(sq->outnet->infra, &sq->addr, 
+			sq->addrlen, -1, time(0)))
+			log_err("Out of memory caching no edns for host");
+		sq->status = serviced_query_TCP;
+		serviced_tcp_initiate(sq->outnet, sq, c->buffer);
+		return 0;
+	}
+
+	(void)rbtree_delete(sq->outnet->serviced, sq);
+	serviced_callbacks(sq, error, c);
+	serviced_delete(sq);
+	return 0;
+}
+
+static void
+serviced_tcp_initiate(struct outside_network* outnet, 
+	struct serviced_query* sq, ldns_buffer* buff)
+{
+	serviced_encode(sq, buff, sq->status == serviced_query_TCP_EDNS);
+	sq->pending = pending_tcp_query(outnet, buff, &sq->addr,
+		sq->addrlen, TCP_QUERY_TIMEOUT, serviced_tcp_callback, 
+		sq, outnet->rnd);
+	if(!sq->pending) {
+		/* delete from tree so that a retry by above layer does not
+		 * clash with this entry */
+		log_err("serviced_tcp_initiate: failed to send tcp query");
+		(void)rbtree_delete(outnet->serviced, sq);
+		serviced_callbacks(sq, NETEVENT_CLOSED, NULL);
+		serviced_delete(sq);
+	}
+}
+
+static int 
+serviced_udp_callback(struct comm_point* c, void* arg, int error,
+        struct comm_reply* ATTR_UNUSED(rep))
+{
+	struct serviced_query* sq = (struct serviced_query*)arg;
+	struct outside_network* outnet = sq->outnet;
+	time_t now = time(NULL);
+	int roundtime;
+	sq->pending = NULL; /* removed after callback */
+	if(error == NETEVENT_TIMEOUT) {
+		sq->retry++;
+		if(!infra_rtt_update(outnet->infra, &sq->addr, sq->addrlen,
+			-1, now))
+			log_err("out of memory in UDP exponential backoff");
+		if(sq->retry < OUTBOUND_UDP_RETRY) {
+			if(!serviced_udp_send(sq, c->buffer)) {
+				(void)rbtree_delete(outnet->serviced, sq);
+				serviced_callbacks(sq, NETEVENT_CLOSED, c);
+				serviced_delete(sq);
+			}
+			return 0;
+		}
+		error = NETEVENT_TIMEOUT;
+		/* UDP does not work, fallback to TCP below */
+	}
+	if(error == NETEVENT_NOERROR && sq->status == serviced_query_UDP_EDNS 
+		&& LDNS_RCODE_WIRE(ldns_buffer_begin(c->buffer)) 
+			== LDNS_RCODE_FORMERR) {
+		/* note no EDNS, fallback without EDNS */
+		if(!infra_edns_update(outnet->infra, &sq->addr, sq->addrlen,
+			-1, now)) {
+			log_err("Out of memory caching no edns for host");
+		}
+		sq->status = serviced_query_UDP;
+		sq->retry = 0;
+		if(!serviced_udp_send(sq, c->buffer)) {
+			(void)rbtree_delete(outnet->serviced, sq);
+			serviced_callbacks(sq, NETEVENT_CLOSED, c);
+			serviced_delete(sq);
+		}
+		return 0;
+	}
+	if(error != NETEVENT_NOERROR || 
+		LDNS_TC_WIRE(ldns_buffer_begin(c->buffer))) {
+		/* fallback to TCP */
+		/* this discards partial UDP contents */
+		if(sq->status == serviced_query_UDP_EDNS)
+			sq->status = serviced_query_TCP_EDNS;
+		else	sq->status = serviced_query_TCP;
+		serviced_tcp_initiate(outnet, sq, c->buffer);
+		return 0;
+	}
+	/* yay! an answer */
+	roundtime = (int)now - (int)sq->last_sent_time;
+	if(roundtime >= 0)
+		if(!infra_rtt_update(outnet->infra, &sq->addr, sq->addrlen, 
+			roundtime*1000, now))
+			log_err("out of memory noting rtt.");
+	(void)rbtree_delete(outnet->serviced, sq);
+	serviced_callbacks(sq, error, c);
+	serviced_delete(sq);
+	return 0;
+}
+
+struct serviced_query* 
+outnet_serviced_query(struct outside_network* outnet,
+	uint8_t* qname, size_t qnamelen, uint16_t qtype, uint16_t qclass,
+	uint16_t flags, int dnssec, struct sockaddr_storage* addr,
+	socklen_t addrlen, comm_point_callback_t* callback,
+	void* callback_arg, ldns_buffer* buff)
+{
+	struct serviced_query* sq;
+	struct service_callback* cb;
+	serviced_gen_query(buff, qname, qnamelen, qtype, qclass, flags);
+	sq = lookup_serviced(outnet, buff, dnssec, addr, addrlen);
+	if(!(cb = (struct service_callback*)malloc(sizeof(*cb))))
+		return NULL;
+	if(!sq) {
+		/* make new serviced query entry */
+		sq = serviced_create(outnet, buff, dnssec, addr, addrlen);
+		if(!sq) {
+			free(cb);
+			return NULL;
+		}
+		/* perform first network action */
+		if(!serviced_udp_send(sq, buff)) {
+			free(sq);
+			free(cb);
+			return NULL;
+		}
+	}
+	/* add callback to list of callbacks */
+	cb->cb = callback;
+	cb->cb_arg = callback_arg;
+	cb->next = sq->cblist;
+	sq->cblist = cb;
+
+	return sq;
+}
+
+/** remove callback from list */
+static void
+callback_list_remove(struct serviced_query* sq, void* cb_arg)
+{
+	struct service_callback** pp = &sq->cblist;
+	while(*pp) {
+		if((*pp)->cb_arg == cb_arg) {
+			struct service_callback* del = *pp;
+			*pp = del->next;
+			free(del);
+			return;
+		}
+		pp = &(*pp)->next;
+	}
+}
+
+void outnet_serviced_query_stop(struct serviced_query* sq, void* cb_arg)
+{
+	if(!sq) 
+		return;
+	callback_list_remove(sq, cb_arg);
+	if(!sq->cblist) {
+		(void)rbtree_delete(sq->outnet->serviced, sq);
+		serviced_delete(sq);
+	}
+}
--- a/services/outside_network.h
+++ b/services/outside_network.h
@ -51,6 +51,7 @@ struct pending_timeout;
 struct ub_randstate;
 struct pending_tcp;
 struct waiting_tcp;
+struct infra_cache;

 /**
 * Send queries to outside servers and wait for answers from servers.
@ -68,19 +69,25 @@ struct outside_network {
 	 * Array of udp comm point* that are used to listen to pending events.
 	 * Each is on a different port. This is for ip4 ports.
 	 */
-	struct comm_point **udp4_ports;
+	struct comm_point** udp4_ports;
 	/** number of udp4 ports */
 	size_t num_udp4;

 	/**
 	 * The opened ip6 ports.
 	 */
-	struct comm_point **udp6_ports;
+	struct comm_point** udp6_ports;
 	/** number of udp6 ports */
 	size_t num_udp6;

 	/** pending udp answers. sorted by id, addr */
-	rbtree_t *pending;
+	rbtree_t* pending;
+	/** serviced queries, sorted by qbuf, addr, dnssec */
+	rbtree_t* serviced;
+	/** host cache, pointer but not owned by outnet. */
+	struct infra_cache* infra;
+	/** where to get random numbers */
+	struct ub_randstate* rnd;

 	/**
 	 * Array of tcp pending used for outgoing TCP connections.
@ -170,6 +177,64 @@ struct waiting_tcp {
 	void* cb_arg;
 };

+/**
+ * Callback to party interested in serviced query results.
+ */
+struct service_callback {
+	/** next in callback list */
+	struct service_callback* next;
+	/** callback function */
+	comm_point_callback_t* cb;
+	/** user argument for callback function */
+	void* cb_arg;
+};
+
+/**
+ * Query service record.
+ * Contains query and destination. UDP, TCP, EDNS are all tried.
+ * complete with retries and timeouts. A number of interested parties can
+ * receive a callback.
+ */
+struct serviced_query {
+	/** The rbtree node, key is this record */
+	rbnode_t node;
+	/** The query that needs to be answered. Starts with flags u16,
+	 * then qdcount, ..., including qname, qtype, qclass. Does not include
+	 * EDNS record. */
+	uint8_t* qbuf;
+	/** length of qbuf. */
+	size_t qbuflen;
+	/** If an EDNS section is included, the DO bit will be turned on. */
+	int dnssec;
+	/** where to send it */
+	struct sockaddr_storage addr;
+	/** length of addr field in use. */
+	socklen_t addrlen;
+	/** current status */
+	enum serviced_query_status {
+		/** initial status */
+		serviced_initial,
+		/** UDP with EDNS sent */
+		serviced_query_UDP_EDNS,
+		/** UDP without EDNS sent */
+		serviced_query_UDP,
+		/** TCP with EDNS sent */
+		serviced_query_TCP_EDNS,
+		/** TCP without EDNS sent */
+		serviced_query_TCP
+	} status;
+	/** number of UDP retries */
+	int retry;
+	/** time last UDP was sent */
+	time_t last_sent_time;
+	/** outside network this is part of */
+	struct outside_network* outnet;
+	/** list of interested parties that need callback on results. */
+	struct service_callback* cblist;
+	/** the UDP or TCP query that is pending, see status which */
+	void* pending;
+};
+
 /**
 * Create outside_network structure with N udp ports.
 * @param base: the communication base to use for event handling.
@ -183,11 +248,14 @@ struct waiting_tcp {
 * @param port_base: if -1 system assigns ports, otherwise try to get
 *    the ports numbered from this starting number.
 * @param num_tcp: number of outgoing tcp buffers to preallocate.
+ * @param infra: pointer to infra cached used for serviced queries.
+ * @param rnd: stored to create random numbers for serviced queries.
 * @return: the new structure (with no pending answers) or NULL on error.
 */
 struct outside_network* outside_network_create(struct comm_base* base,
 	size_t bufsize, size_t num_ports, char** ifs, int num_ifs,
-	int do_ip4, int do_ip6, int port_base, size_t num_tcp);
+	int do_ip4, int do_ip6, int port_base, size_t num_tcp, 
+	struct infra_cache* infra, struct ub_randstate* rnd);

 /**
 * Delete outside_network structure.
@ -206,12 +274,12 @@ void outside_network_delete(struct outside_network* outnet);
 * @param callback: function to call on error, timeout or reply.
 * @param callback_arg: user argument for callback function.
 * @param rnd: random state for generating ID and port.
- * @return: false on error for malloc or socket.
+ * @return: NULL on error for malloc or socket. Else the pending query object.
 */
-int pending_udp_query(struct outside_network* outnet, ldns_buffer* packet, 
-	struct sockaddr_storage* addr, socklen_t addrlen, int timeout,
-	comm_point_callback_t* callback, void* callback_arg,
-	struct ub_randstate* rnd);
+struct pending* pending_udp_query(struct outside_network* outnet, 
+	ldns_buffer* packet, struct sockaddr_storage* addr, 
+	socklen_t addrlen, int timeout, comm_point_callback_t* callback, 
+	void* callback_arg, struct ub_randstate* rnd);

 /**
 * Send TCP query. May wait for TCP buffer. Selects ID to be random, and 
@ -226,12 +294,12 @@ int pending_udp_query(struct outside_network* outnet, ldns_buffer* packet,
 * @param callback: function to call on error, timeout or reply.
 * @param callback_arg: user argument for callback function.
 * @param rnd: random state for generating ID.
- * @return: false on error for malloc or socket.
+ * @return: false on error for malloc or socket. Else the pending TCP object.
 */
-int pending_tcp_query(struct outside_network* outnet, ldns_buffer* packet, 
-	struct sockaddr_storage* addr, socklen_t addrlen, int timeout,
-	comm_point_callback_t* callback, void* callback_arg,
-	struct ub_randstate* rnd);
+struct waiting_tcp* pending_tcp_query(struct outside_network* outnet, 
+	ldns_buffer* packet, struct sockaddr_storage* addr, 
+	socklen_t addrlen, int timeout, comm_point_callback_t* callback, 
+	void* callback_arg, struct ub_randstate* rnd);

 /**
 * Delete pending answer.
@ -241,4 +309,37 @@ int pending_tcp_query(struct outside_network* outnet, ldns_buffer* packet,
 */
 void pending_delete(struct outside_network* outnet, struct pending* p);

+/**
+ * Perform a serviced query to the authoritative servers.
+ * Duplicate efforts are detected, and EDNS, TCP and UDP retry is performed.
+ * @param outnet: outside network, with rbtree of serviced queries.
+ * @param qname: what qname to query.
+ * @param qnamelen: length of qname in octets including 0 root label.
+ * @param qtype: rrset type to query (host format)
+ * @param qclass: query class. (host format)
+ * @param flags: flags u16 (host format), includes opcode, CD bit.
+ * @param dnssec: if set, DO bit is set in EDNS queries.
+ * @param callback: callback function.
+ * @param callback_arg: user argument to callback function.
+ * @param addr: to which server to send the query.
+ * @param addrlen: length of addr.
+ * @param buff: scratch buffer to create query contents in. Empty on exit.
+ * @return 0 on error, or pointer to serviced query that is used to answer
+ *	this serviced query may be shared with other callbacks as well.
+ */
+struct serviced_query* outnet_serviced_query(struct outside_network* outnet,
+	uint8_t* qname, size_t qnamelen, uint16_t qtype, uint16_t qclass,
+	uint16_t flags, int dnssec, struct sockaddr_storage* addr, 
+	socklen_t addrlen, comm_point_callback_t* callback, 
+	void* callback_arg, ldns_buffer* buff);
+
+/**
+ * Remove service query callback.
+ * If that leads to zero callbacks, the query is completely cancelled.
+ * @param sq: serviced query to adjust.
+ * @param cb_arg: callback argument of callback that needs removal.
+ *	same as the callback_arg to outnet_serviced_query().
+ */
+void outnet_serviced_query_stop(struct serviced_query* sq, void* cb_arg);
+
 #endif /* OUTSIDE_NETWORK_H */
--- a/testcode/fake_event.c
+++ b/testcode/fake_event.c
@ -642,7 +642,8 @@ outside_network_create(struct comm_base* base, size_t bufsize,
 	size_t ATTR_UNUSED(num_ports), char** ATTR_UNUSED(ifs), 
 	int ATTR_UNUSED(num_ifs), int ATTR_UNUSED(do_ip4), 
 	int ATTR_UNUSED(do_ip6), int ATTR_UNUSED(port_base),
-	size_t ATTR_UNUSED(num_tcp))
+	size_t ATTR_UNUSED(num_tcp), struct infra_cache* ATTR_UNUSED(infra),
+	struct ub_randstate* ATTR_UNUSED(rnd))
 {
 	struct outside_network* outnet =  calloc(1, 
 		sizeof(struct outside_network));
@ -664,7 +665,7 @@ outside_network_delete(struct outside_network* outnet)
 	free(outnet);
 }

-int 
+struct pending* 
 pending_udp_query(struct outside_network* outnet, ldns_buffer* packet,
 	struct sockaddr_storage* addr, socklen_t addrlen, int timeout,
 	comm_point_callback_t* callback, void* callback_arg,
@ -709,10 +710,10 @@ pending_udp_query(struct outside_network* outnet, ldns_buffer* packet,
 	/* add to list */
 	pend->next = runtime->pending_list;
 	runtime->pending_list = pend;
-	return 1;
+	return (struct pending*)pend;
 }

-int 
+struct waiting_tcp* 
 pending_tcp_query(struct outside_network* outnet, ldns_buffer* packet,
 	struct sockaddr_storage* addr, socklen_t addrlen, int timeout,
 	comm_point_callback_t* callback, void* callback_arg,
@ -757,7 +758,7 @@ pending_tcp_query(struct outside_network* outnet, ldns_buffer* packet,
 	/* add to list */
 	pend->next = runtime->pending_list;
 	runtime->pending_list = pend;
-	return 1;
+	return (struct waiting_tcp*)pend;
 }

 struct listen_port* listening_ports_open(struct config_file* ATTR_UNUSED(cfg))
--- a/util/data/msgparse.h
+++ b/util/data/msgparse.h
@ -189,8 +189,6 @@ struct rr_parse {
 #define EDNS_RCODE_BADVERS	16	/** bad EDNS version */
 /** largest valid compression offset */
 #define PTR_MAX_OFFSET 	0x3fff
-/** bits for EDNS bitfield */
-#define EDNS_DO	0x8000 /* Dnssec Ok */

 /**
 * EDNS data storage
--- a/util/net_help.h
+++ b/util/net_help.h
@ -60,6 +60,8 @@
 #define EDNS_ADVERTISED_VERSION         0
 /** Advertised size of EDNS capabilities */
 #define EDNS_ADVERTISED_SIZE    4096
+/** bits for EDNS bitfield */
+#define EDNS_DO 0x8000 /* Dnssec Ok */

 /**
 * See if string is ip4 or ip6.
--- a/util/storage/lruhash.h
+++ b/util/storage/lruhash.h
@ -96,6 +96,12 @@
 *	  the scenario requires more than three threads.
 * 	o so the queue length is 3 threads in a bad situation. The fourth is
 *	  unable to use the hashtable.
+ *
+ * If you need to acquire locks on multiple items from the hashtable.
+ *	o you MUST release all locks on items from the hashtable before
+ *	  doing the next lookup/insert/delete/whatever.
+ *	o To acquire multiple items you should use a special routine that
+ *	  obtains the locks on those multiple items in one go.
 */

 #ifndef UTIL_STORAGE_LRUHASH_H