Merge branch 'master' into zonemd

doc/Changelog

@@ -1,3 +1,21 @@
+15 October 2020: Wouter
+	- Merge PR #326 from netblue30: DoH: implement content-length
+	  header field
+	- DoH content length, simplify code, remove declaration after
+	  statement and fix cast warning.
+
+14 October 2020: Wouter
+	- Fix for python reply callback to see mesh state reply_list member,
+	  it only removes it briefly for the commpoint call so that it does
+	  not drop it and attempt to modify the reply list during reply.
+	- Fix that if there are on reply callbacks, those are called per
+	  reply and a new message created if that was modified by the call.
+	- Free up auth zone parse region after use for lookup of host
+
+13 October 2020: Wouter
+	- Fix #323: unbound testsuite fails on mock build in systemd-nspawn
+	  if systemd support is build.
+
 9 October 2020: Wouter
 	- Fix dnstap socket and the chroot not applied properly to the dnstap
 	  socket path.

services/authzone.c

@@ -5490,6 +5490,7 @@ void auth_xfer_transfer_lookup_callback(void* arg, int rcode, sldns_buffer* buf,
 				verbose(VERB_ALGO, "auth zone %s host %s type %s transfer lookup has no answer", zname, xfr->task_transfer->lookup_target->host, (xfr->task_transfer->lookup_aaaa?"AAAA":"A"));
 			}
 		}
+		regional_free_all(temp);
 	} else {
 		if(verbosity >= VERB_ALGO) {
 			char zname[255+1];
@@ -6547,6 +6548,7 @@ void auth_xfer_probe_lookup_callback(void* arg, int rcode, sldns_buffer* buf,
 				verbose(VERB_ALGO, "auth zone %s host %s type %s probe lookup has no address", zname, xfr->task_probe->lookup_target->host, (xfr->task_probe->lookup_aaaa?"AAAA":"A"));
 			}
 		}
+		regional_free_all(temp);
 	} else {
 		if(verbosity >= VERB_ALGO) {
 			char zname[255+1];

services/listen_dnsport.c

@@ -2177,9 +2177,10 @@ int http2_submit_dns_response(struct http2_session* h2_session)
 	int ret;
 	nghttp2_data_provider data_prd;
 	char status[4];
-	nghttp2_nv headers[2];
+	nghttp2_nv headers[3];
 	struct http2_stream* h2_stream = h2_session->c->h2_stream;
 	size_t rlen;
+	char rlen_str[32];
 
 	if(h2_stream->rbuffer) {
 		log_err("http2 submit response error: rbuffer already "
@@ -2198,6 +2199,8 @@ int http2_submit_dns_response(struct http2_session* h2_session)
 	}
 
 	rlen = sldns_buffer_remaining(h2_session->c->buffer);
+	snprintf(rlen_str, sizeof(rlen_str), "%u", rlen);
+
 	lock_basic_lock(&http2_response_buffer_count_lock);
 	if(http2_response_buffer_count + rlen > http2_response_buffer_max) {
 		lock_basic_unlock(&http2_response_buffer_count_lock);
@@ -2228,13 +2231,11 @@ int http2_submit_dns_response(struct http2_session* h2_session)
 	headers[1].valuelen = 23;
 	headers[1].flags = NGHTTP2_NV_FLAG_NONE;
 
-	/*TODO be nice and add the content-length header
 	headers[2].name = (uint8_t*)"content-length";
 	headers[2].namelen = 14;
-	headers[2].value = 
-	headers[2].valuelen = 
+	headers[2].value = (uint8_t*)rlen_str;
+	headers[2].valuelen = strlen(rlen_str);
 	headers[2].flags = NGHTTP2_NV_FLAG_NONE;
-	*/
 
 	sldns_buffer_write(h2_stream->rbuffer,
 		sldns_buffer_current(h2_session->c->buffer),
@@ -2244,7 +2245,7 @@ int http2_submit_dns_response(struct http2_session* h2_session)
 	data_prd.source.ptr = h2_session;
 	data_prd.read_callback = http2_submit_response_read_callback;
 	ret = nghttp2_submit_response(h2_session->session, h2_stream->stream_id,
-		headers, 2, &data_prd);
+		headers, 3, &data_prd);
 	if(ret) {
 		verbose(VERB_QUERY, "http2: set_stream_user_data failed, "
 			"error: %s", nghttp2_strerror(ret));

services/mesh.c

@@ -1196,6 +1196,12 @@ mesh_send_reply(struct mesh_state* m, int rcode, struct reply_info* rep,
 	/* Copy the client's EDNS for later restore, to make sure the edns
 	 * compare is with the correct edns options. */
 	struct edns_data edns_bak = r->edns;
+	/* briefly set the replylist to null in case the
+	 * meshsendreply calls tcpreqinfo sendreply that
+	 * comm_point_drops because of size, and then the
+	 * null stops the mesh state remove and thus
+	 * reply_list modification and accounting */
+	struct mesh_reply* rlist = m->reply_list;
 	/* examine security status */
 	if(m->s.env->need_to_validate && (!(r->qflags&BIT_CD) ||
 		m->s.env->cfg->ignore_cd) && rep && 
@@ -1229,14 +1235,16 @@ mesh_send_reply(struct mesh_state* m, int rcode, struct reply_info* rep,
 		prev->edns.bits == r->edns.bits && 
 		prev->edns.udp_size == r->edns.udp_size &&
 		edns_opt_list_compare(prev->edns.opt_list, r->edns.opt_list)
-		== 0) {
+		== 0 && !m->s.env->inplace_cb_lists[inplace_cb_reply]) {
 		/* if the previous reply is identical to this one, fix ID */
 		if(prev_buffer != r_buffer)
 			sldns_buffer_copy(r_buffer, prev_buffer);
 		sldns_buffer_write_at(r_buffer, 0, &r->qid, sizeof(uint16_t));
 		sldns_buffer_write_at(r_buffer, 12, r->qname,
 			m->s.qinfo.qname_len);
+		m->reply_list = NULL;
 		comm_point_send_reply(&r->query_reply);
+		m->reply_list = rlist;
 	} else if(rcode) {
 		m->s.qinfo.qname = r->qname;
 		m->s.qinfo.local_alias = r->local_alias;
@@ -1251,7 +1259,9 @@ mesh_send_reply(struct mesh_state* m, int rcode, struct reply_info* rep,
 		}
 		error_encode(r_buffer, rcode, &m->s.qinfo, r->qid,
 			r->qflags, &r->edns);
+		m->reply_list = NULL;
 		comm_point_send_reply(&r->query_reply);
+		m->reply_list = rlist;
 	} else {
 		size_t udp_size = r->edns.udp_size;
 		r->edns.edns_version = EDNS_ADVERTISED_VERSION;
@@ -1277,7 +1287,9 @@ mesh_send_reply(struct mesh_state* m, int rcode, struct reply_info* rep,
 				&m->s.qinfo, r->qid, r->qflags, &r->edns);
 		}
 		r->edns = edns_bak;
+		m->reply_list = NULL;
 		comm_point_send_reply(&r->query_reply);
+		m->reply_list = rlist;
 	}
 	/* account */
 	log_assert(m->s.env->mesh->num_reply_addrs > 0);
@@ -1365,20 +1377,12 @@ void mesh_query_done(struct mesh_state* mstate)
 			mstate->reply_list = reply_list;
 		} else {
 			struct sldns_buffer* r_buffer = r->query_reply.c->buffer;
-			struct mesh_reply* rlist = mstate->reply_list;
 			if(r->query_reply.c->tcp_req_info) {
 				r_buffer = r->query_reply.c->tcp_req_info->spool_buffer;
 				prev_buffer = NULL;
 			}
-			/* briefly set the replylist to null in case the
-			 * meshsendreply calls tcpreqinfo sendreply that
-			 * comm_point_drops because of size, and then the
-			 * null stops the mesh state remove and thus
-			 * reply_list modification and accounting */
-			mstate->reply_list = NULL;
 			mesh_send_reply(mstate, mstate->s.return_rcode, rep,
 				r, r_buffer, prev, prev_buffer);
-			mstate->reply_list = rlist;
 			if(r->query_reply.c->tcp_req_info) {
 				tcp_req_info_remove_mesh_state(r->query_reply.c->tcp_req_info, mstate);
 				r_buffer = NULL;
@@ -1894,7 +1898,7 @@ mesh_serve_expired_callback(void* arg)
 {
 	struct mesh_state* mstate = (struct mesh_state*) arg;
 	struct module_qstate* qstate = &mstate->s;
-	struct mesh_reply* r, *rlist;
+	struct mesh_reply* r;
 	struct mesh_area* mesh = qstate->env->mesh;
 	struct dns_msg* msg;
 	struct mesh_cb* c;
@@ -1999,15 +2003,8 @@ mesh_serve_expired_callback(void* arg)
 		r_buffer = r->query_reply.c->buffer;
 		if(r->query_reply.c->tcp_req_info)
 			r_buffer = r->query_reply.c->tcp_req_info->spool_buffer;
-		/* briefly set the replylist to null in case the meshsendreply
-		 * calls tcpreqinfo sendreply that comm_point_drops because
-		 * of size, and then the null stops the mesh state remove and
-		 * thus reply_list modification and accounting */
-		rlist = mstate->reply_list;
-		mstate->reply_list = NULL;
 		mesh_send_reply(mstate, LDNS_RCODE_NOERROR, msg->rep,
 			r, r_buffer, prev, prev_buffer);
-		mstate->reply_list = rlist;
 		if(r->query_reply.c->tcp_req_info)
 			tcp_req_info_remove_mesh_state(r->query_reply.c->tcp_req_info, mstate);
 		prev = r;

testcode/do-tests.sh

@@ -29,6 +29,9 @@ else
 	HAVE_MINGW=no
 fi
 
+# stop tests from notifying systemd, if that is compiled in.
+export -n NOTIFY_SOCKET
+
 cd testdata;
 sh ../testcode/mini_tdir.sh clean
 rm -f .perfstats.txt

testcode/run_vm.sh

@@ -40,6 +40,8 @@ cleanup() {
 	exit 0
 }
 trap cleanup INT
+# stop tests from notifying systemd, if that is compiled in.
+export -n NOTIFY_SOCKET
 
 for t in $RUNLIST
 do

testcode/testbound.c

@@ -362,6 +362,10 @@ main(int argc, char* argv[])
 	/* we do not want the test to depend on the timezone */
 	(void)putenv("TZ=UTC");
 	memset(pass_argv, 0, sizeof(pass_argv));
+#ifdef HAVE_SYSTEMD
+	/* we do not want the test to use systemd daemon startup notification*/
+	(void)unsetenv("NOTIFY_SOCKET");
+#endif /* HAVE_SYSTEMD */
 
 	log_init(NULL, 0, NULL);
 	/* determine commandline options for the daemon */