- so-sndbuf option for very busy servers, a bit like so-rcvbuf.

git-svn-id: file:///svn/unbound/trunk@2344 be551aaa-1e26-0410-a405-d3ace91eadb9

doc/Changelog

@@ -1,6 +1,7 @@
 15 November 2010: Wouter
 	- silence 'tcp connect: broken pipe' and 'net down' at low verbosity.
 	- iana portlist updated.
+	- so-sndbuf option for very busy servers, a bit like so-rcvbuf.
 
 9 November 2010: Wouter
 	- unbound-anchor compiles with openssl 0.9.7.

doc/example.conf.in

@@ -79,6 +79,10 @@ server:
 	# 0 is system default.  Use 4m to catch query spikes for busy servers.
 	# so-rcvbuf: 0
 
+	# buffer size for UDP port 53 outgoing (SO_SNDBUF socket option).
+	# 0 is system default.  Use 4m to handle spikes on very busy servers.
+	# so-sndbuf: 0
+
 	# EDNS reassembly buffer to advertise to UDP peers (the actual buffer
 	# is set with msg-buffer-size). 1480 can solve fragmentation (timeouts).
 	# edns-buffer-size: 4096

doc/unbound.conf.5.in

@@ -231,6 +231,17 @@ net.core.rmem_max.  On BSD change kern.ipc.maxsockbuf in /etc/sysctl.conf.
 On OpenBSD change header and recompile kernel. On Solaris ndd \-set
 /dev/udp udp_max_buf 8388608.
 .TP
+.B so\-sndbuf: \fI<number>
+If not 0, then set the SO_SNDBUF socket option to get more buffer space on
+UDP port 53 outgoing queries.  This for very busy servers handles spikes
+in answer traffic, otherwise 'send: resource temporarily unavailable'
+can get logged, the buffer overrun is also visible by netstat \-su.
+Default is 0 (use system value).  Specify the number of bytes to ask
+for, try "4m" on a very busy server.  The OS caps it at a maximum, on
+linux unbound needs root permission to bypass the limit, or the admin
+can use sysctl net.core.wmem_max.  On BSD, Solaris changes are similar
+to so\-rcvbuf.
+.TP
 .B rrset\-cache\-size: \fI<number>
 Number of bytes size of the RRset cache. Default is 4 megabytes.
 A plain number is in bytes, append 'k', 'm' or 'g' for kilobytes, megabytes

services/listen_dnsport.c

@@ -90,7 +90,8 @@ verbose_print_addr(struct addrinfo *addr)
 
 int
 create_udp_sock(int family, int socktype, struct sockaddr* addr,
-        socklen_t addrlen, int v6only, int* inuse, int* noproto, int rcv)
+        socklen_t addrlen, int v6only, int* inuse, int* noproto,
+	int rcv, int snd)
 {
 	int s;
 #if defined(IPV6_USE_MIN_MTU)
@@ -102,6 +103,9 @@ create_udp_sock(int family, int socktype, struct sockaddr* addr,
 #if !defined(SO_RCVBUFFORCE) && !defined(SO_RCVBUF)
 	(void)rcv;
 #endif
+#if !defined(SO_SNDBUFFORCE) && !defined(SO_SNDBUF)
+	(void)snd;
+#endif
 #ifndef IPV6_V6ONLY
 	(void)v6only;
 #endif
@@ -182,6 +186,65 @@ create_udp_sock(int family, int socktype, struct sockaddr* addr,
 		}
 #  endif
 #endif /* SO_RCVBUF */
+	}
+	/* first do RCVBUF as the receive buffer is more important */
+	if(snd) {
+#ifdef SO_SNDBUF
+		int got;
+		socklen_t slen = (socklen_t)sizeof(got);
+#  ifdef SO_SNDBUFFORCE
+		/* Linux specific: try to use root permission to override
+		 * system limits on sndbuf. The limit is stored in 
+		 * /proc/sys/net/core/wmem_max or sysctl net.core.wmem_max */
+		if(setsockopt(s, SOL_SOCKET, SO_SNDBUFFORCE, (void*)&snd, 
+			(socklen_t)sizeof(snd)) < 0) {
+			if(errno != EPERM) {
+#    ifndef USE_WINSOCK
+				log_err("setsockopt(..., SO_SNDBUFFORCE, "
+					"...) failed: %s", strerror(errno));
+				close(s);
+#    else
+				log_err("setsockopt(..., SO_SNDBUFFORCE, "
+					"...) failed: %s", 
+					wsa_strerror(WSAGetLastError()));
+				closesocket(s);
+#    endif
+				*noproto = 0;
+				*inuse = 0;
+				return -1;
+			}
+#  endif /* SO_SNDBUFFORCE */
+			if(setsockopt(s, SOL_SOCKET, SO_SNDBUF, (void*)&snd, 
+				(socklen_t)sizeof(snd)) < 0) {
+#  ifndef USE_WINSOCK
+				log_err("setsockopt(..., SO_SNDBUF, "
+					"...) failed: %s", strerror(errno));
+				close(s);
+#  else
+				log_err("setsockopt(..., SO_SNDBUF, "
+					"...) failed: %s", 
+					wsa_strerror(WSAGetLastError()));
+				closesocket(s);
+#  endif
+				*noproto = 0;
+				*inuse = 0;
+				return -1;
+			}
+			/* check if we got the right thing or if system
+			 * reduced to some system max.  Warn if so */
+			if(getsockopt(s, SOL_SOCKET, SO_SNDBUF, (void*)&got, 
+				&slen) >= 0 && got < snd/2) {
+				log_warn("so-sndbuf %u was not granted. "
+					"Got %u. To fix: start with "
+					"root permissions(linux) or sysctl "
+					"bigger net.core.wmem_max(linux) or "
+					"kern.ipc.maxsockbuf(bsd) values.",
+					(unsigned)snd, (unsigned)got);
+			}
+#  ifdef SO_SNDBUFFORCE
+		}
+#  endif
+#endif /* SO_SNDBUF */
 	}
 	if(family == AF_INET6) {
 # if defined(IPV6_V6ONLY)
@@ -394,7 +457,7 @@ create_tcp_accept_sock(struct addrinfo *addr, int v6only, int* noproto)
  */
 static int
 make_sock(int stype, const char* ifname, const char* port, 
-	struct addrinfo *hints, int v6only, int* noip6, size_t rcv)
+	struct addrinfo *hints, int v6only, int* noip6, size_t rcv, size_t snd)
 {
 	struct addrinfo *res = NULL;
 	int r, s, inuse, noproto;
@@ -420,8 +483,8 @@ make_sock(int stype, const char* ifname, const char* port,
 	if(stype == SOCK_DGRAM) {
 		verbose_print_addr(res);
 		s = create_udp_sock(res->ai_family, res->ai_socktype,
-			(struct sockaddr*)res->ai_addr,
-			res->ai_addrlen, v6only, &inuse, &noproto, (int)rcv);
+			(struct sockaddr*)res->ai_addr, res->ai_addrlen,
+			v6only, &inuse, &noproto, (int)rcv, (int)snd);
 		if(s == -1 && inuse) {
 			log_err("bind: address already in use");
 		} else if(s == -1 && noproto && hints->ai_family == AF_INET6){
@@ -440,7 +503,7 @@ make_sock(int stype, const char* ifname, const char* port,
 /** make socket and first see if ifname contains port override info */
 static int
 make_sock_port(int stype, const char* ifname, const char* port, 
-	struct addrinfo *hints, int v6only, int* noip6, size_t rcv)
+	struct addrinfo *hints, int v6only, int* noip6, size_t rcv, size_t snd)
 {
 	char* s = strchr(ifname, '@');
 	if(s) {
@@ -461,9 +524,10 @@ make_sock_port(int stype, const char* ifname, const char* port,
 		newif[s-ifname] = 0;
 		strncpy(p, s+1, sizeof(p));
 		p[strlen(s+1)]=0;
-		return make_sock(stype, newif, p, hints, v6only, noip6, rcv);
+		return make_sock(stype, newif, p, hints, v6only, noip6,
+			rcv, snd);
 	}
-	return make_sock(stype, ifname, port, hints, v6only, noip6, rcv);
+	return make_sock(stype, ifname, port, hints, v6only, noip6, rcv, snd);
 }
 
 /**
@@ -553,19 +617,20 @@ set_recvpktinfo(int s, int family)
  * @param port: Port number to use (as string).
  * @param list: list of open ports, appended to, changed to point to list head.
  * @param rcv: receive buffer size for UDP
+ * @param snd: send buffer size for UDP
  * @return: returns false on error.
  */
 static int
 ports_create_if(const char* ifname, int do_auto, int do_udp, int do_tcp, 
 	struct addrinfo *hints, const char* port, struct listen_port** list,
-	size_t rcv)
+	size_t rcv, size_t snd)
 {
 	int s, noip6=0;
 	if(!do_udp && !do_tcp)
 		return 0;
 	if(do_auto) {
 		if((s = make_sock_port(SOCK_DGRAM, ifname, port, hints, 1, 
-			&noip6, rcv)) == -1) {
+			&noip6, rcv, snd)) == -1) {
 			if(noip6) {
 				log_warn("IPv6 protocol not available");
 				return 1;
@@ -586,7 +651,7 @@ ports_create_if(const char* ifname, int do_auto, int do_udp, int do_tcp,
 	} else if(do_udp) {
 		/* regular udp socket */
 		if((s = make_sock_port(SOCK_DGRAM, ifname, port, hints, 1, 
-			&noip6, rcv)) == -1) {
+			&noip6, rcv, snd)) == -1) {
 			if(noip6) {
 				log_warn("IPv6 protocol not available");
 				return 1;
@@ -604,7 +669,7 @@ ports_create_if(const char* ifname, int do_auto, int do_udp, int do_tcp,
 	}
 	if(do_tcp) {
 		if((s = make_sock_port(SOCK_STREAM, ifname, port, hints, 1, 
-			&noip6, 0)) == -1) {
+			&noip6, 0, 0)) == -1) {
 			if(noip6) {
 				/*log_warn("IPv6 protocol not available");*/
 				return 1;
@@ -750,7 +815,8 @@ listening_ports_open(struct config_file* cfg)
 			hints.ai_family = AF_INET6;
 			if(!ports_create_if(do_auto?"::0":"::1", 
 				do_auto, cfg->do_udp, do_tcp, 
-				&hints, portbuf, &list, cfg->socket_rcvbuf)) {
+				&hints, portbuf, &list,
+				cfg->so_rcvbuf, cfg->so_sndbuf)) {
 				listening_ports_free(list);
 				return NULL;
 			}
@@ -759,7 +825,8 @@ listening_ports_open(struct config_file* cfg)
 			hints.ai_family = AF_INET;
 			if(!ports_create_if(do_auto?"0.0.0.0":"127.0.0.1", 
 				do_auto, cfg->do_udp, do_tcp, 
-				&hints, portbuf, &list, cfg->socket_rcvbuf)) {
+				&hints, portbuf, &list,
+				cfg->so_rcvbuf, cfg->so_sndbuf)) {
 				listening_ports_free(list);
 				return NULL;
 			}
@@ -771,7 +838,7 @@ listening_ports_open(struct config_file* cfg)
 			hints.ai_family = AF_INET6;
 			if(!ports_create_if(cfg->ifs[i], 0, cfg->do_udp, 
 				do_tcp, &hints, portbuf, &list, 
-				cfg->socket_rcvbuf)) {
+				cfg->so_rcvbuf, cfg->so_sndbuf)) {
 				listening_ports_free(list);
 				return NULL;
 			}
@@ -781,7 +848,7 @@ listening_ports_open(struct config_file* cfg)
 			hints.ai_family = AF_INET;
 			if(!ports_create_if(cfg->ifs[i], 0, cfg->do_udp, 
 				do_tcp, &hints, portbuf, &list, 
-				cfg->socket_rcvbuf)) {
+				cfg->so_rcvbuf, cfg->so_sndbuf)) {
 				listening_ports_free(list);
 				return NULL;
 			}

services/listen_dnsport.h

@@ -162,10 +162,12 @@ size_t listen_get_mem(struct listen_dnsport* listen);
  * @param noproto: on error, this is set true if cause is that the
 	IPv6 proto (family) is not available.
  * @param rcv: set size on rcvbuf with socket option, if 0 it is not set.
+ * @param snd: set size on sndbuf with socket option, if 0 it is not set.
  * @return: the socket. -1 on error.
  */
 int create_udp_sock(int family, int socktype, struct sockaddr* addr, 
-	socklen_t addrlen, int v6only, int* inuse, int* noproto, int rcv);
+	socklen_t addrlen, int v6only, int* inuse, int* noproto, int rcv,
+	int snd);
 
 /**
  * Create and bind TCP listening socket

services/outside_network.c

@@ -820,12 +820,14 @@ udp_sockport(struct sockaddr_storage* addr, socklen_t addrlen, int port,
 		struct sockaddr_in6* sa = (struct sockaddr_in6*)addr;
 		sa->sin6_port = (in_port_t)htons((uint16_t)port);
 		fd = create_udp_sock(AF_INET6, SOCK_DGRAM, 
-			(struct sockaddr*)addr, addrlen, 1, inuse, &noproto, 0);
+			(struct sockaddr*)addr, addrlen, 1, inuse, &noproto,
+			0, 0);
 	} else {
 		struct sockaddr_in* sa = (struct sockaddr_in*)addr;
 		sa->sin_port = (in_port_t)htons((uint16_t)port);
 		fd = create_udp_sock(AF_INET, SOCK_DGRAM, 
-			(struct sockaddr*)addr, addrlen, 1, inuse, &noproto, 0);
+			(struct sockaddr*)addr, addrlen, 1, inuse, &noproto,
+			0, 0);
 	}
 	return fd;
 }

util/config_file.c

@@ -140,7 +140,8 @@ config_create(void)
 	cfg->root_hints = NULL;
 	cfg->do_daemonize = 1;
 	cfg->if_automatic = 0;
-	cfg->socket_rcvbuf = 0;
+	cfg->so_rcvbuf = 0;
+	cfg->so_sndbuf = 0;
 	cfg->num_ifs = 0;
 	cfg->ifs = NULL;
 	cfg->num_out_ifs = 0;
@@ -334,7 +335,8 @@ int config_set_option(struct config_file* cfg, const char* opt,
 	else S_POW2("msg-cache-slabs:", msg_cache_slabs)
 	else S_SIZET_NONZERO("num-queries-per-thread:",num_queries_per_thread)
 	else S_SIZET_OR_ZERO("jostle-timeout:", jostle_time)
-	else S_MEMSIZE("so-rcvbuf:", socket_rcvbuf)
+	else S_MEMSIZE("so-rcvbuf:", so_rcvbuf)
+	else S_MEMSIZE("so-sndbuf:", so_sndbuf)
 	else S_MEMSIZE("rrset-cache-size:", rrset_cache_size)
 	else S_POW2("rrset-cache-slabs:", rrset_cache_slabs)
 	else S_YNO("prefetch:", prefetch)
@@ -553,7 +555,8 @@ config_get_option(struct config_file* cfg, const char* opt,
 	else O_DEC(opt, "msg-cache-slabs", msg_cache_slabs)
 	else O_DEC(opt, "num-queries-per-thread", num_queries_per_thread)
 	else O_UNS(opt, "jostle-timeout", jostle_time)
-	else O_MEM(opt, "so-rcvbuf", socket_rcvbuf)
+	else O_MEM(opt, "so-rcvbuf", so_rcvbuf)
+	else O_MEM(opt, "so-sndbuf", so_sndbuf)
 	else O_MEM(opt, "rrset-cache-size", rrset_cache_size)
 	else O_DEC(opt, "rrset-cache-slabs", rrset_cache_slabs)
 	else O_YNO(opt, "prefetch-key", prefetch_key)

util/config_file.h

@@ -120,7 +120,9 @@ struct config_file {
 	 * and recvmsg/sendmsg ancillary data to detect interfaces, boolean */
 	int if_automatic;
 	/** SO_RCVBUF size to set on port 53 UDP socket */
-	size_t socket_rcvbuf;
+	size_t so_rcvbuf;
+	/** SO_SNDBUF size to set on port 53 UDP socket */
+	size_t so_sndbuf;
 
 	/** number of interfaces to open. If 0 default all interfaces. */
 	int num_ifs;

util/configlexer.lex

@@ -141,6 +141,7 @@ interface{COLON}		{ YDVAR(1, VAR_INTERFACE) }
 outgoing-interface{COLON}	{ YDVAR(1, VAR_OUTGOING_INTERFACE) }
 interface-automatic{COLON}	{ YDVAR(1, VAR_INTERFACE_AUTOMATIC) }
 so-rcvbuf{COLON}		{ YDVAR(1, VAR_SO_RCVBUF) }
+so-sndbuf{COLON}		{ YDVAR(1, VAR_SO_SNDBUF) }
 chroot{COLON}			{ YDVAR(1, VAR_CHROOT) }
 username{COLON}			{ YDVAR(1, VAR_USERNAME) }
 directory{COLON}		{ YDVAR(1, VAR_DIRECTORY) }

util/configparser.h

@@ -152,7 +152,8 @@
      VAR_SO_RCVBUF = 368,
      VAR_EDNS_BUFFER_SIZE = 369,
      VAR_PREFETCH = 370,
-     VAR_PREFETCH_KEY = 371
+     VAR_PREFETCH_KEY = 371,
+     VAR_SO_SNDBUF = 372
    };
 #endif
 /* Tokens.  */
@@ -270,6 +271,7 @@
 #define VAR_EDNS_BUFFER_SIZE 369
 #define VAR_PREFETCH 370
 #define VAR_PREFETCH_KEY 371
+#define VAR_SO_SNDBUF 372
 
 
 
@@ -286,7 +288,7 @@ typedef union YYSTYPE
 
 
 /* Line 1676 of yacc.c  */
-#line 290 "util/configparser.h"
+#line 292 "util/configparser.h"
 } YYSTYPE;
 # define YYSTYPE_IS_TRIVIAL 1
 # define yystype YYSTYPE /* obsolescent; will be withdrawn */

util/configparser.y

@@ -101,7 +101,7 @@ extern struct config_parser_state* cfg_parser;
 %token VAR_VAL_SIG_SKEW_MAX VAR_CACHE_MIN_TTL VAR_VAL_LOG_LEVEL
 %token VAR_AUTO_TRUST_ANCHOR_FILE VAR_KEEP_MISSING VAR_ADD_HOLDDOWN 
 %token VAR_DEL_HOLDDOWN VAR_SO_RCVBUF VAR_EDNS_BUFFER_SIZE VAR_PREFETCH
-%token VAR_PREFETCH_KEY
+%token VAR_PREFETCH_KEY VAR_SO_SNDBUF
 
 %%
 toplevelvars: /* empty */ | toplevelvars toplevelvar ;
@@ -154,7 +154,8 @@ content_server: server_num_threads | server_verbosity | server_port |
 	server_val_sig_skew_max | server_cache_min_ttl | server_val_log_level |
 	server_auto_trust_anchor_file | server_add_holddown | 
 	server_del_holddown | server_keep_missing | server_so_rcvbuf |
-	server_edns_buffer_size | server_prefetch | server_prefetch_key
+	server_edns_buffer_size | server_prefetch | server_prefetch_key |
+	server_so_sndbuf
 	;
 stubstart: VAR_STUB_ZONE
 	{
@@ -524,7 +525,15 @@ server_version: VAR_VERSION STRING_ARG
 server_so_rcvbuf: VAR_SO_RCVBUF STRING_ARG
 	{
 		OUTYY(("P(server_so_rcvbuf:%s)\n", $2));
-		if(!cfg_parse_memsize($2, &cfg_parser->cfg->socket_rcvbuf))
+		if(!cfg_parse_memsize($2, &cfg_parser->cfg->so_rcvbuf))
+			yyerror("buffer size expected");
+		free($2);
+	}
+	;
+server_so_sndbuf: VAR_SO_SNDBUF STRING_ARG
+	{
+		OUTYY(("P(server_so_sndbuf:%s)\n", $2));
+		if(!cfg_parse_memsize($2, &cfg_parser->cfg->so_sndbuf))
 			yyerror("buffer size expected");
 		free($2);
 	}