diff --git a/CHANGES b/CHANGES index 6f97f24edf..367a14341a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,6 @@ +5551. [bug] Only assign threads to CPUs in the CPU affinity set. + Thanks to Ole Bjørn Hessen. [GL #2245] + 5550. [func] Print a warning when falling back to the "increment" SOA serial method. [GL #2058] diff --git a/bin/tests/system/Makefile.am b/bin/tests/system/Makefile.am index 4b61d7a20b..a90f0b6278 100644 --- a/bin/tests/system/Makefile.am +++ b/bin/tests/system/Makefile.am @@ -90,6 +90,7 @@ TESTS += \ checkconf \ checknames \ checkzone \ + cpu \ database \ dlz \ dlzexternal \ diff --git a/bin/tests/system/conf.sh.in b/bin/tests/system/conf.sh.in index 48442c405c..74b0ebbf1f 100644 --- a/bin/tests/system/conf.sh.in +++ b/bin/tests/system/conf.sh.in @@ -124,7 +124,7 @@ PERL=$(command -v "@PERL@") # Windows process management leave empty PSSUSPEND= -PYTHON=$(command -v "@PYTHON@") +PYTHON=$(command -v "@PYTHON@" || true) PYTEST=@PYTEST@ # diff --git a/bin/tests/system/cpu/clean.sh b/bin/tests/system/cpu/clean.sh new file mode 100644 index 0000000000..0a6c10bb75 --- /dev/null +++ b/bin/tests/system/cpu/clean.sh @@ -0,0 +1,15 @@ +#!/bin/sh +# +# Copyright (C) Internet Systems Consortium, Inc. ("ISC") +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at https://mozilla.org/MPL/2.0/. +# +# See the COPYRIGHT file distributed with this work for additional +# information regarding copyright ownership. + +set -e + +rm -f ps.out +rm -f ns1/named.conf ns1/managed-keys.* ns1/named.run ns1/named.memstats diff --git a/bin/tests/system/cpu/ns1/named.conf.in b/bin/tests/system/cpu/ns1/named.conf.in new file mode 100644 index 0000000000..d152b2709d --- /dev/null +++ b/bin/tests/system/cpu/ns1/named.conf.in @@ -0,0 +1,18 @@ +/* + * Copyright (C) Internet Systems Consortium, Inc. ("ISC") + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * + * See the COPYRIGHT file distributed with this work for additional + * information regarding copyright ownership. + */ + +options { + query-source address 10.53.0.1; + port @PORT@; + pid-file "named.pid"; + listen-on { 10.53.0.1; }; + listen-on-v6 { none; }; +}; diff --git a/bin/tests/system/cpu/prereq.sh b/bin/tests/system/cpu/prereq.sh new file mode 100644 index 0000000000..8b0cd6e804 --- /dev/null +++ b/bin/tests/system/cpu/prereq.sh @@ -0,0 +1,30 @@ +#!/bin/sh -e +# +# Copyright (C) Internet Systems Consortium, Inc. ("ISC") +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at https://mozilla.org/MPL/2.0/. +# +# See the COPYRIGHT file distributed with this work for additional +# information regarding copyright ownership. + +set -e + +# shellcheck source=conf.sh +. ../conf.sh + +case $(uname) in + Linux*) + ;; + *) + echo_i "cpu test only runs on Linux, skipping test" + exit 255 + ;; +esac + +# TASKSET will be an empty string if no taskset program was found. +TASKSET=$(command -v "taskset" || true) +if ! test -x "$TASKSET" ; then + exit 255 +fi diff --git a/bin/tests/system/cpu/setup.sh b/bin/tests/system/cpu/setup.sh new file mode 100644 index 0000000000..5d4286829d --- /dev/null +++ b/bin/tests/system/cpu/setup.sh @@ -0,0 +1,19 @@ +#!/bin/sh -e +# +# Copyright (C) Internet Systems Consortium, Inc. ("ISC") +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at https://mozilla.org/MPL/2.0/. +# +# See the COPYRIGHT file distributed with this work for additional +# information regarding copyright ownership. + +# shellcheck source=conf.sh +. ../conf.sh + +set -e + +$SHELL clean.sh + +copy_setports ns1/named.conf.in ns1/named.conf diff --git a/bin/tests/system/cpu/tests.sh b/bin/tests/system/cpu/tests.sh new file mode 100644 index 0000000000..54cdb0abfc --- /dev/null +++ b/bin/tests/system/cpu/tests.sh @@ -0,0 +1,46 @@ +#!/bin/sh +# +# Copyright (C) Internet Systems Consortium, Inc. ("ISC") +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at https://mozilla.org/MPL/2.0/. +# +# See the COPYRIGHT file distributed with this work for additional +# information regarding copyright ownership. + +# shellcheck source=conf.sh +. ../conf.sh + +status=0 +n=0 + +n=$((n+1)) +echo_i "stop server ($n)" +ret=0 +$PERL ../stop.pl cpu ns1 || ret=1 +test "$ret" -eq 0 || echo_i "failed" +status=$((status+ret)) + +n=$((n+1)) +echo_i "start server with taskset ($n)" +ret=0 +start_server --noclean --taskset fff0 --restart --port "${PORT}" cpu ns1 || ret=1 +test "$ret" -eq 0 || echo_i "failed" +status=$((status+ret)) + +n=$((n+1)) +echo_i "check ps output ($n)" +ret=0 +ps -T -o pid,psr,time,comm -e > ps.out +pid=$(cat ns1/named.pid) +echo_i "pid=$pid" +psr=$(awk -v pid="$pid" '$1 == pid && $4 == "isc-net-0000" {print $2}' < ps.out) +echo_i "psr=$psr" +# The next available cpu relative to the existing affinity mask is 4. +test "$psr" -eq 4 || ret=1 +test "$ret" -eq 0 || echo_i "failed" +status=$((status+ret)) + +echo_i "exit status: $status" +[ $status -eq 0 ] || exit 1 diff --git a/bin/tests/system/start.pl b/bin/tests/system/start.pl index af64569e7c..54d345069f 100755 --- a/bin/tests/system/start.pl +++ b/bin/tests/system/start.pl @@ -23,7 +23,7 @@ use Getopt::Long; use Time::HiRes 'sleep'; # allows sleeping fractional seconds # Usage: -# perl start.pl [--noclean] [--restart] [--port port] test [server [options]] +# perl start.pl [--noclean] [--restart] [--port port] [--taskset cpus] test [server [options]] # # --noclean Do not cleanup files in server directory. # @@ -38,6 +38,10 @@ use Time::HiRes 'sleep'; # allows sleeping fractional seconds # of the file "named.port" in the server directory containing # the number of the query port.) # +# --taskset cpus Use taskset to signal which cpus can be used. For example +# cpus=fff0 means all cpus aexcept for 0, 1, 2, and 3 are +# eligible. +# # test Name of the test directory. # # server Name of the server directory. This will be of the form @@ -57,15 +61,17 @@ use Time::HiRes 'sleep'; # allows sleeping fractional seconds # the file is ignored). If "options" is already set, then # "named.args" is ignored. -my $usage = "usage: $0 [--noclean] [--restart] [--port ] test-directory [server-directory [server-options]]"; +my $usage = "usage: $0 [--noclean] [--restart] [--port ] [--taskset ] test-directory [server-directory [server-options]]"; my $clean = 1; my $restart = 0; my $queryport = 5300; +my $taskset = ""; GetOptions( - 'clean!' => \$clean, - 'restart!' => \$restart, - 'port=i' => \$queryport, + 'clean!' => \$clean, + 'restart!' => \$restart, + 'port=i' => \$queryport, + 'taskset=s' => \$taskset, ) or die "$usage\n"; my( $test, $server_arg, $options_arg ) = @ARGV; @@ -232,7 +238,11 @@ sub construct_ns_command { $command .= "$NAMED -m none -M external "; } else { - $command = "$NAMED "; + if ($taskset) { + $command = "taskset $taskset $NAMED "; + } else { + $command = "$NAMED "; + } } my $args_file = $testdir . "/" . $server . "/" . "named.args"; diff --git a/doc/notes/notes-current.rst b/doc/notes/notes-current.rst index d1e4717483..c26e11c919 100644 --- a/doc/notes/notes-current.rst +++ b/doc/notes/notes-current.rst @@ -46,4 +46,6 @@ Feature Changes Bug Fixes ~~~~~~~~~ -- None. +- Only assign threads to CPUs in the CPU affinity set, so that ``named`` no + longer attempts to run threads on CPUs outside the affinity set. Thanks to + Ole Bjørn Hessen. [GL #2245] diff --git a/lib/isc/pthreads/thread.c b/lib/isc/pthreads/thread.c index b17b36d2d6..6477aa876f 100644 --- a/lib/isc/pthreads/thread.c +++ b/lib/isc/pthreads/thread.c @@ -128,40 +128,92 @@ isc_thread_yield(void) { #endif /* if defined(HAVE_SCHED_YIELD) */ } +#if defined(HAVE_CPUSET_SETAFFINITY) || defined(HAVE_PTHREAD_SETAFFINITY_NP) +#if defined(HAVE_CPUSET_SETAFFINITY) +static int +getaffinity(cpuset_t *set) { + return (cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, -1, + sizeof(*set), set)); +} +static int +issetaffinity(int cpu, cpuset_t *set) { + return ((cpu >= CPU_SETSIZE) ? -1 : CPU_ISSET(cpu, set) ? 1 : 0); +} +static int +setaffinity(int cpu, cpuset_t *set) { + CPU_ZERO(set); + CPU_SET(cpu, set); + return (cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, -1, + sizeof(*set), set)); +} +#elif defined(__NetBSD__) +static int +getaffinity(cpuset_t *set) { + return (pthread_getaffinity_np(pthread_self(), cpuset_size(set), set)); +} +static int +issetaffinity(int cpu, cpuset_t *set) { + return (cpuset_isset(cpu, set)); +} +static int +setaffinity(int cpu, cpuset_t *set) { + cpuset_zero(set); + cpuset_set(cpu, set); + return (pthread_setaffinity_np(pthread_self(), cpuset_size(set), set)); +} +#else /* linux ? */ +static int +getaffinity(cpu_set_t *set) { + return (pthread_getaffinity_np(pthread_self(), sizeof(*set), set)); +} +static int +issetaffinity(int cpu, cpu_set_t *set) { + return ((cpu >= CPU_SETSIZE) ? -1 : CPU_ISSET(cpu, set) ? 1 : 0); +} +static int +setaffinity(int cpu, cpu_set_t *set) { + CPU_ZERO(set); + CPU_SET(cpu, set); + return (pthread_setaffinity_np(pthread_self(), sizeof(*set), set)); +} +#endif +#endif + isc_result_t isc_thread_setaffinity(int cpu) { +#if defined(HAVE_CPUSET_SETAFFINITY) || defined(HAVE_PTHREAD_SETAFFINITY_NP) + int cpu_id = -1, cpu_aff_ok_counter = -1, n; #if defined(HAVE_CPUSET_SETAFFINITY) - cpuset_t cpuset; - CPU_ZERO(&cpuset); - CPU_SET(cpu, &cpuset); - if (cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, -1, - sizeof(cpuset), &cpuset) != 0) - { + cpuset_t _set, *set = &_set; +#define cpuset_destroy(x) ((void)0) +#elif defined(__NetBSD__) + cpuset_t *set = cpuset_create(); + if (set == NULL) { return (ISC_R_FAILURE); } -#elif defined(HAVE_PTHREAD_SETAFFINITY_NP) -#if defined(__NetBSD__) - cpuset_t *cset; - cset = cpuset_create(); - if (cset == NULL) { +#else /* linux? */ + cpu_set_t _set, *set = &_set; +#define cpuset_destroy(x) ((void)0) +#endif + + if (getaffinity(set) != 0) { + cpuset_destroy(set); return (ISC_R_FAILURE); } - cpuset_set(cpu, cset); - if (pthread_setaffinity_np(pthread_self(), cpuset_size(cset), cset) != - 0) { - cpuset_destroy(cset); + while (cpu_aff_ok_counter < cpu) { + cpu_id++; + if ((n = issetaffinity(cpu_id, set)) > 0) { + cpu_aff_ok_counter++; + } else if (n < 0) { + cpuset_destroy(set); + return (ISC_R_FAILURE); + } + } + if (setaffinity(cpu_id, set) != 0) { + cpuset_destroy(set); return (ISC_R_FAILURE); } - cpuset_destroy(cset); -#else /* linux? */ - cpu_set_t set; - CPU_ZERO(&set); - CPU_SET(cpu, &set); - if (pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &set) != - 0) { - return (ISC_R_FAILURE); - } -#endif /* __NetBSD__ */ + cpuset_destroy(set); #elif defined(HAVE_PROCESSOR_BIND) if (processor_bind(P_LWPID, P_MYID, cpu, NULL) != 0) { return (ISC_R_FAILURE); diff --git a/util/copyrights b/util/copyrights index 6d21735927..a7055d0da9 100644 --- a/util/copyrights +++ b/util/copyrights @@ -290,6 +290,10 @@ ./bin/tests/system/cookie/clean.sh SH 2014,2015,2016,2018,2019,2020 ./bin/tests/system/cookie/setup.sh SH 2018,2019,2020 ./bin/tests/system/cookie/tests.sh SH 2014,2015,2016,2017,2018,2019,2020 +./bin/tests/system/cpu/clean.sh SH 2020 +./bin/tests/system/cpu/prereq.sh SH 2020 +./bin/tests/system/cpu/setup.sh SH 2020 +./bin/tests/system/cpu/tests.sh SH 2020 ./bin/tests/system/custom-test-driver X 2020 ./bin/tests/system/database/clean.sh SH 2011,2012,2014,2016,2018,2019,2020 ./bin/tests/system/database/setup.sh SH 2011,2012,2016,2018,2019,2020