[master] Improve handling of TCP_FASTOPEN on FreeBSD

4726.	[port]		Prevent setsockopt() errors related to TCP_FASTOPEN
			from being logged on FreeBSD if the kernel does not
			support it.  Notify the user when the kernel does
			support TCP_FASTOPEN, but it is disabled by sysctl.
			Add a new configure option, --disable-tcp-fastopen, to
			disable use of TCP_FASTOPEN altogether. [RT #44754]

CHANGES

@@ -1,3 +1,10 @@
+4726.	[port]		Prevent setsockopt() errors related to TCP_FASTOPEN
+			from being logged on FreeBSD if the kernel does not
+			support it.  Notify the user when the kernel does
+			support TCP_FASTOPEN, but it is disabled by sysctl.
+			Add a new configure option, --disable-tcp-fastopen, to
+			disable use of TCP_FASTOPEN altogether. [RT #44754]
+
 4725.	[bug]		Nsupdate: "recvsoa" was incorrectly reported for
 			failures in sending the update message.  The correct
 			location to be reported is "update_completed".

configure.in

@@ -3361,10 +3361,18 @@ AC_TRY_COMPILE([
 AC_SUBST(ISC_PLATFORM_NEEDPORTT)
 
 #
-# Look for TCP_FASTOPEN
+# Allow forcibly disabling TCP Fast Open support as autodetection might yield
+# confusing results on some systems (e.g. FreeBSD; see set_tcp_fastopen()
+# comment in lib/isc/unix/socket.c).
 #
+
+AC_ARG_ENABLE(tcp_fastopen,
+	[  --disable-tcp-fastopen  disable TCP Fast Open support [[default=autodetect]]])
+
 AC_MSG_CHECKING(for TCP_FASTOPEN socket option)
-AC_EGREP_CPP(has_tfo, [
+case "$enable_tcp_fastopen" in
+yes|''|autodetect)
+	AC_EGREP_CPP(has_tfo, [
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <netinet/tcp.h>
@@ -3372,10 +3380,17 @@ AC_EGREP_CPP(has_tfo, [
 int has_tfo() { return (0); }
 #endif
 ],
-	[AC_MSG_RESULT(yes)
-	ISC_PLATFORM_HAVETFO="#define ISC_PLATFORM_HAVETFO 1"],
-	[AC_MSG_RESULT(no)
-	ISC_PLATFORM_HAVETFO="#undef ISC_PLATFORM_HAVETFO"])
+		[AC_MSG_RESULT(yes)
+		ISC_PLATFORM_HAVETFO="#define ISC_PLATFORM_HAVETFO 1"],
+		[AC_MSG_RESULT(no)
+		ISC_PLATFORM_HAVETFO="#undef ISC_PLATFORM_HAVETFO"])
+
+		;;
+	no)
+		AC_MSG_RESULT(disabled)
+		ISC_PLATFORM_HAVETFO="#undef ISC_PLATFORM_HAVETFO"
+		;;
+esac
 AC_SUBST(ISC_PLATFORM_HAVETFO)
 
 #

doc/misc/tcp-fast-open

@@ -19,6 +19,10 @@ System specific notes:
  - FreeBSD doesn't interpret the argument as a queue length but
   only as an on/off switch.
 
+ - Using TCP Fast Open on FreeBSD, as of versions 10.3 and 11.0, requires
+   compiling a custom kernel and setting the "net.inet.tcp.fastopen.enabled"
+   sysctl to 1.
+
  - Apple OS X/macOS allows only 0 or 1 so the code puts 1 for this system.
 
  - Windows 10 uses a 0/1 char flag? Note that TCP_FASTOPEN is defined

lib/isc/unix/socket.c

@@ -14,6 +14,9 @@
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <sys/stat.h>
+#ifdef HAVE_SYS_SYSCTL_H
+#include <sys/sysctl.h>
+#endif
 #include <sys/time.h>
 #include <sys/uio.h>
 
@@ -5649,6 +5652,69 @@ isc__socket_filter(isc_socket_t *sock0, const char *filter) {
 #endif
 }
 
+/*
+ * Try enabling TCP Fast Open for a given socket if the OS supports it.
+ */
+static void
+set_tcp_fastopen(isc__socket_t *sock, unsigned int backlog) {
+#if defined(ISC_PLATFORM_HAVETFO) && defined(TCP_FASTOPEN)
+	char strbuf[ISC_STRERRORSIZE];
+
+/*
+ * FreeBSD, as of versions 10.3 and 11.0, defines TCP_FASTOPEN while also
+ * shipping a default kernel without TFO support, so we special-case it by
+ * performing an additional runtime check for TFO support using sysctl to
+ * prevent setsockopt() errors from being logged.
+ */
+#if defined(__FreeBSD__) && defined(HAVE_SYSCTLBYNAME)
+#define SYSCTL_TFO "net.inet.tcp.fastopen.enabled"
+	unsigned int enabled;
+	size_t enabledlen = sizeof(enabled);
+	static isc_boolean_t tfo_notice_logged = ISC_FALSE;
+
+	if (sysctlbyname(SYSCTL_TFO, &enabled, &enabledlen, NULL, 0) < 0) {
+		/*
+		 * This kernel does not support TCP Fast Open.  There is
+		 * nothing more we can do.
+		 */
+		return;
+	} else if (enabled == 0) {
+		/*
+		 * This kernel does support TCP Fast Open, but it is disabled
+		 * by sysctl.  Notify the user, but do not nag.
+		 */
+		if (!tfo_notice_logged) {
+			isc_log_write(isc_lctx, ISC_LOGCATEGORY_GENERAL,
+				      ISC_LOGMODULE_SOCKET, ISC_LOG_NOTICE,
+				      "TCP_FASTOPEN support is disabled by "
+				      "sysctl (" SYSCTL_TFO " = 0)");
+			tfo_notice_logged = ISC_TRUE;
+		}
+		return;
+	}
+#endif
+
+#ifdef __APPLE__
+	backlog = 1;
+#else
+	backlog = backlog / 2;
+	if (backlog == 0)
+		backlog = 1;
+#endif
+	if (setsockopt(sock->fd, IPPROTO_TCP, TCP_FASTOPEN,
+		       (void *)&backlog, sizeof(backlog)) < 0) {
+		isc__strerror(errno, strbuf, sizeof(strbuf));
+		UNEXPECTED_ERROR(__FILE__, __LINE__,
+				 "setsockopt(%d, TCP_FASTOPEN) failed with %s",
+				 sock->fd, strbuf);
+		/* TCP_FASTOPEN is experimental so ignore failures */
+	}
+#else
+	UNUSED(sock);
+	UNUSED(backlog);
+#endif
+}
+
 /*
  * Set up to listen on a given socket.  We do this by creating an internal
  * event that will be dispatched when the socket has read activity.  The
@@ -5685,23 +5751,7 @@ isc__socket_listen(isc_socket_t *sock0, unsigned int backlog) {
 		return (ISC_R_UNEXPECTED);
 	}
 
-#if defined(ISC_PLATFORM_HAVETFO) && defined(TCP_FASTOPEN)
-#ifdef __APPLE__
-	backlog = 1;
-#else
-	backlog = backlog / 2;
-	if (backlog == 0)
-		backlog = 1;
-#endif
-	if (setsockopt(sock->fd, IPPROTO_TCP, TCP_FASTOPEN,
-		       (void *)&backlog, sizeof(backlog)) < 0) {
-		isc__strerror(errno, strbuf, sizeof(strbuf));
-		UNEXPECTED_ERROR(__FILE__, __LINE__,
-				 "setsockopt(%d, TCP_FASTOPEN) failed with %s",
-				 sock->fd, strbuf);
-		/* TCP_FASTOPEN is experimental so ignore failures */
-	}
-#endif
+	set_tcp_fastopen(sock, backlog);
 
 	sock->listener = 1;