Merge branch '2245-bind-9-16-8-does-not-honor-cpu-affinity' into 'main'

Resolve "bind 9.16.8 does not honor CPU affinity"

Closes #2245

See merge request isc-projects/bind9!4395

CHANGES

@@ -1,3 +1,6 @@
+5551.	[bug]		Only assign threads to CPUs in the CPU affinity set.
+			Thanks to Ole Bjørn Hessen. [GL #2245]
+
 5550.	[func]		Print a warning when falling back to the "increment" SOA
 			serial method. [GL #2058]
 

bin/tests/system/Makefile.am

@@ -90,6 +90,7 @@ TESTS +=			\
 	checkconf		\
 	checknames		\
 	checkzone		\
+	cpu			\
 	database		\
 	dlz			\
 	dlzexternal		\

bin/tests/system/conf.sh.in

@@ -124,7 +124,7 @@ PERL=$(command -v "@PERL@")
 # Windows process management leave empty
 PSSUSPEND=
 
-PYTHON=$(command -v "@PYTHON@")
+PYTHON=$(command -v "@PYTHON@" || true)
 PYTEST=@PYTEST@
 
 #

bin/tests/system/cpu/clean.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+set -e
+
+rm -f ps.out
+rm -f ns1/named.conf ns1/managed-keys.* ns1/named.run ns1/named.memstats

bin/tests/system/cpu/ns1/named.conf.in

@@ -0,0 +1,18 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+options {
+	query-source address 10.53.0.1;
+	port @PORT@;
+	pid-file "named.pid";
+	listen-on { 10.53.0.1; };
+	listen-on-v6 { none; };
+};

bin/tests/system/cpu/prereq.sh

@@ -0,0 +1,30 @@
+#!/bin/sh -e
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+set -e
+
+# shellcheck source=conf.sh
+. ../conf.sh
+
+case $(uname) in
+	Linux*)
+		;;
+	*)
+		echo_i "cpu test only runs on Linux, skipping test"
+		exit 255
+		;;
+esac
+
+# TASKSET will be an empty string if no taskset program was found.
+TASKSET=$(command -v "taskset" || true)
+if ! test -x "$TASKSET" ; then
+	exit 255
+fi

bin/tests/system/cpu/setup.sh

@@ -0,0 +1,19 @@
+#!/bin/sh -e
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+# shellcheck source=conf.sh
+. ../conf.sh
+
+set -e
+
+$SHELL clean.sh
+
+copy_setports ns1/named.conf.in ns1/named.conf

bin/tests/system/cpu/tests.sh

@@ -0,0 +1,46 @@
+#!/bin/sh
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+# shellcheck source=conf.sh
+. ../conf.sh
+
+status=0
+n=0
+
+n=$((n+1))
+echo_i "stop server ($n)"
+ret=0
+$PERL ../stop.pl cpu ns1 || ret=1
+test "$ret" -eq 0 || echo_i "failed"
+status=$((status+ret))
+
+n=$((n+1))
+echo_i "start server with taskset ($n)"
+ret=0
+start_server --noclean --taskset fff0 --restart --port "${PORT}" cpu ns1 || ret=1
+test "$ret" -eq 0 || echo_i "failed"
+status=$((status+ret))
+
+n=$((n+1))
+echo_i "check ps output ($n)"
+ret=0
+ps -T -o pid,psr,time,comm -e > ps.out
+pid=$(cat ns1/named.pid)
+echo_i "pid=$pid"
+psr=$(awk -v pid="$pid" '$1 == pid && $4 == "isc-net-0000" {print $2}' < ps.out)
+echo_i "psr=$psr"
+# The next available cpu relative to the existing affinity mask is 4.
+test "$psr" -eq 4 || ret=1
+test "$ret" -eq 0 || echo_i "failed"
+status=$((status+ret))
+
+echo_i "exit status: $status"
+[ $status -eq 0 ] || exit 1

bin/tests/system/start.pl

@@ -23,7 +23,7 @@ use Getopt::Long;
 use Time::HiRes 'sleep'; # allows sleeping fractional seconds
 
 # Usage:
-#   perl start.pl [--noclean] [--restart] [--port port] test [server [options]]
+#   perl start.pl [--noclean] [--restart] [--port port] [--taskset cpus] test [server [options]]
 #
 #   --noclean       Do not cleanup files in server directory.
 #
@@ -38,6 +38,10 @@ use Time::HiRes 'sleep'; # allows sleeping fractional seconds
 #                   of the file "named.port" in the server directory containing
 #                   the number of the query port.)
 #
+#   --taskset cpus  Use taskset to signal which cpus can be used. For example
+#                   cpus=fff0 means all cpus aexcept for 0, 1, 2, and 3 are
+#                   eligible.
+#
 #   test            Name of the test directory.
 #
 #   server          Name of the server directory.  This will be of the form
@@ -57,15 +61,17 @@ use Time::HiRes 'sleep'; # allows sleeping fractional seconds
 #                   the file is ignored). If "options" is already set, then
 #                   "named.args" is ignored.
 
-my $usage = "usage: $0 [--noclean] [--restart] [--port <port>] test-directory [server-directory [server-options]]";
+my $usage = "usage: $0 [--noclean] [--restart] [--port <port>] [--taskset <cpus>] test-directory [server-directory [server-options]]";
 my $clean = 1;
 my $restart = 0;
 my $queryport = 5300;
+my $taskset = "";
 
 GetOptions(
-	'clean!'   => \$clean,
-	'restart!' => \$restart,
-	'port=i'   => \$queryport,
+	'clean!'    => \$clean,
+	'restart!'  => \$restart,
+	'port=i'    => \$queryport,
+	'taskset=s' => \$taskset,
 ) or die "$usage\n";
 
 my( $test, $server_arg, $options_arg ) = @ARGV;
@@ -232,7 +238,11 @@ sub construct_ns_command {
 
 		$command .= "$NAMED -m none -M external ";
 	} else {
-		$command = "$NAMED ";
+		if ($taskset) {
+			$command = "taskset $taskset $NAMED ";
+		} else {
+			$command = "$NAMED ";
+		}
 	}
 
 	my $args_file = $testdir . "/" . $server . "/" . "named.args";

doc/notes/notes-current.rst

@@ -46,4 +46,6 @@ Feature Changes
 Bug Fixes
 ~~~~~~~~~
 
-- None.
+- Only assign threads to CPUs in the CPU affinity set, so that ``named`` no
+  longer attempts to run threads on CPUs outside the affinity set. Thanks to
+  Ole Bjørn Hessen. [GL #2245]

lib/isc/pthreads/thread.c

@@ -128,40 +128,92 @@ isc_thread_yield(void) {
 #endif /* if defined(HAVE_SCHED_YIELD) */
 }
 
+#if defined(HAVE_CPUSET_SETAFFINITY) || defined(HAVE_PTHREAD_SETAFFINITY_NP)
+#if defined(HAVE_CPUSET_SETAFFINITY)
+static int
+getaffinity(cpuset_t *set) {
+	return (cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, -1,
+				   sizeof(*set), set));
+}
+static int
+issetaffinity(int cpu, cpuset_t *set) {
+	return ((cpu >= CPU_SETSIZE) ? -1 : CPU_ISSET(cpu, set) ? 1 : 0);
+}
+static int
+setaffinity(int cpu, cpuset_t *set) {
+	CPU_ZERO(set);
+	CPU_SET(cpu, set);
+	return (cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, -1,
+				   sizeof(*set), set));
+}
+#elif defined(__NetBSD__)
+static int
+getaffinity(cpuset_t *set) {
+	return (pthread_getaffinity_np(pthread_self(), cpuset_size(set), set));
+}
+static int
+issetaffinity(int cpu, cpuset_t *set) {
+	return (cpuset_isset(cpu, set));
+}
+static int
+setaffinity(int cpu, cpuset_t *set) {
+	cpuset_zero(set);
+	cpuset_set(cpu, set);
+	return (pthread_setaffinity_np(pthread_self(), cpuset_size(set), set));
+}
+#else /* linux ? */
+static int
+getaffinity(cpu_set_t *set) {
+	return (pthread_getaffinity_np(pthread_self(), sizeof(*set), set));
+}
+static int
+issetaffinity(int cpu, cpu_set_t *set) {
+	return ((cpu >= CPU_SETSIZE) ? -1 : CPU_ISSET(cpu, set) ? 1 : 0);
+}
+static int
+setaffinity(int cpu, cpu_set_t *set) {
+	CPU_ZERO(set);
+	CPU_SET(cpu, set);
+	return (pthread_setaffinity_np(pthread_self(), sizeof(*set), set));
+}
+#endif
+#endif
+
 isc_result_t
 isc_thread_setaffinity(int cpu) {
+#if defined(HAVE_CPUSET_SETAFFINITY) || defined(HAVE_PTHREAD_SETAFFINITY_NP)
+	int cpu_id = -1, cpu_aff_ok_counter = -1, n;
 #if defined(HAVE_CPUSET_SETAFFINITY)
-	cpuset_t cpuset;
-	CPU_ZERO(&cpuset);
-	CPU_SET(cpu, &cpuset);
-	if (cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, -1,
-			       sizeof(cpuset), &cpuset) != 0)
-	{
+	cpuset_t _set, *set = &_set;
+#define cpuset_destroy(x) ((void)0)
+#elif defined(__NetBSD__)
+	cpuset_t *set = cpuset_create();
+	if (set == NULL) {
 		return (ISC_R_FAILURE);
 	}
-#elif defined(HAVE_PTHREAD_SETAFFINITY_NP)
-#if defined(__NetBSD__)
-	cpuset_t *cset;
-	cset = cpuset_create();
-	if (cset == NULL) {
+#else /* linux? */
+	cpu_set_t _set, *set = &_set;
+#define cpuset_destroy(x) ((void)0)
+#endif
+
+	if (getaffinity(set) != 0) {
+		cpuset_destroy(set);
 		return (ISC_R_FAILURE);
 	}
-	cpuset_set(cpu, cset);
-	if (pthread_setaffinity_np(pthread_self(), cpuset_size(cset), cset) !=
-	    0) {
-		cpuset_destroy(cset);
+	while (cpu_aff_ok_counter < cpu) {
+		cpu_id++;
+		if ((n = issetaffinity(cpu_id, set)) > 0) {
+			cpu_aff_ok_counter++;
+		} else if (n < 0) {
+			cpuset_destroy(set);
+			return (ISC_R_FAILURE);
+		}
+	}
+	if (setaffinity(cpu_id, set) != 0) {
+		cpuset_destroy(set);
 		return (ISC_R_FAILURE);
 	}
-	cpuset_destroy(cset);
-#else  /* linux? */
-	cpu_set_t set;
-	CPU_ZERO(&set);
-	CPU_SET(cpu, &set);
-	if (pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &set) !=
-	    0) {
-		return (ISC_R_FAILURE);
-	}
-#endif /* __NetBSD__ */
+	cpuset_destroy(set);
 #elif defined(HAVE_PROCESSOR_BIND)
 	if (processor_bind(P_LWPID, P_MYID, cpu, NULL) != 0) {
 		return (ISC_R_FAILURE);

util/copyrights

@@ -290,6 +290,10 @@
 ./bin/tests/system/cookie/clean.sh		SH	2014,2015,2016,2018,2019,2020
 ./bin/tests/system/cookie/setup.sh		SH	2018,2019,2020
 ./bin/tests/system/cookie/tests.sh		SH	2014,2015,2016,2017,2018,2019,2020
+./bin/tests/system/cpu/clean.sh			SH	2020
+./bin/tests/system/cpu/prereq.sh		SH	2020
+./bin/tests/system/cpu/setup.sh			SH	2020
+./bin/tests/system/cpu/tests.sh			SH	2020
 ./bin/tests/system/custom-test-driver		X	2020
 ./bin/tests/system/database/clean.sh		SH	2011,2012,2014,2016,2018,2019,2020
 ./bin/tests/system/database/setup.sh		SH	2011,2012,2016,2018,2019,2020