From 436b76bb17142fc9a9ad9bd420ad61b3e4685b2b Mon Sep 17 00:00:00 2001 From: Tony Finch Date: Wed, 24 Mar 2021 17:52:56 +0100 Subject: [PATCH] Improve the spinloop pause / yield hint Unfortunately, C still lacks a standard function for pause (x86, sparc) or yeild (arm) instructions, for use in spin lock or CAS loops. BIND has its own based on vendor intrinsics or inline asm. Previously, it was buried in the `isc_rwlock` implementation. This commit renames `isc_rwlock_pause()` to `isc_pause()` and moves it into . This commit also fixes the configure script so that it detects ARM yield support on systems that identify as `aarch*` instead of `arm*`. On 64-bit ARM systems we now use the ISB (instruction synchronization barrier) instruction in preference to yield. The ISB instruction pauses the CPU for longer, several nanoseconds, which is more like the x86 pause instruction. There are more details in a Rust pull request, which also refers to MySQL making the same change: https://github.com/rust-lang/rust/pull/84725 --- CHANGES | 3 +++ configure.ac | 4 ++-- lib/isc/Makefile.am | 1 + lib/isc/include/isc/pause.h | 42 +++++++++++++++++++++++++++++++++++++ lib/isc/rwlock.c | 30 ++------------------------ 5 files changed, 50 insertions(+), 30 deletions(-) create mode 100644 lib/isc/include/isc/pause.h diff --git a/CHANGES b/CHANGES index 6a9eee18f5..26cdfad324 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,6 @@ +6097. [port] Improve support for yield / pause instructions in spin + loops on AArch64 platforms. [GL !7469] + 6096. [bug] Fix RPZ reference counting error on shutdown in dns__rpz_timer_cb(). [GL #3866] diff --git a/configure.ac b/configure.ac index c6106cd5cd..4aa1761b26 100644 --- a/configure.ac +++ b/configure.ac @@ -359,8 +359,8 @@ AC_C_VOLATILE # Check for yield support on ARM processors # AS_CASE([$host], - [arm*], - [AC_MSG_CHECKING([for yield instruction support]) + [arm*|aarch64*], + [AC_MSG_CHECKING([for ARM yield instruction support]) AC_COMPILE_IFELSE( [AC_LANG_PROGRAM([[]], [[__asm__ __volatile__ ("yield")]])], diff --git a/lib/isc/Makefile.am b/lib/isc/Makefile.am index 8201e71d6a..b532587cf2 100644 --- a/lib/isc/Makefile.am +++ b/lib/isc/Makefile.am @@ -67,6 +67,7 @@ libisc_la_HEADERS = \ include/isc/once.h \ include/isc/os.h \ include/isc/parseint.h \ + include/isc/pause.h \ include/isc/portset.h \ include/isc/print.h \ include/isc/quota.h \ diff --git a/lib/isc/include/isc/pause.h b/lib/isc/include/isc/pause.h new file mode 100644 index 0000000000..8d7fa57682 --- /dev/null +++ b/lib/isc/include/isc/pause.h @@ -0,0 +1,42 @@ +/* + * Copyright (C) Internet Systems Consortium, Inc. ("ISC") + * + * SPDX-License-Identifier: MPL-2.0 + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, you can obtain one at https://mozilla.org/MPL/2.0/. + * + * See the COPYRIGHT file distributed with this work for additional + * information regarding copyright ownership. + */ + +#pragma once + +#if defined(__x86_64__) +#include +#define isc_pause() _mm_pause() +#elif defined(__i386__) +#define isc_pause() __asm__ __volatile__("rep; nop") +#elif defined(__ia64__) +#define isc_pause() __asm__ __volatile__("hint @pause") +#elif defined(__aarch64__) +#define isc_pause() __asm__ __volatile__("isb") +#elif defined(__arm__) && HAVE_ARM_YIELD +#define isc_pause() __asm__ __volatile__("yield") +#elif defined(sun) && (defined(__sparc) || defined(__sparc__)) +#include +#define isc_pause() smt_pause() +#elif (defined(__sparc) || defined(__sparc__)) && HAVE_SPARC_PAUSE +#define isc_pause() __asm__ __volatile__("pause") +#elif defined(__ppc__) || defined(_ARCH_PPC) || defined(_ARCH_PWR) || \ + defined(_ARCH_PWR2) || defined(_POWER) +#define isc_pause() __asm__ volatile("or 27,27,27") +#else +#define isc_pause() sched_yield() +#endif + +#define isc_pause_n(iters) \ + for (size_t __pause = 0; __pause < iters; __pause++) { \ + isc_pause(); \ + } diff --git a/lib/isc/rwlock.c b/lib/isc/rwlock.c index d00ec7655f..62bdbf1a62 100644 --- a/lib/isc/rwlock.c +++ b/lib/isc/rwlock.c @@ -17,12 +17,9 @@ #include #include -#if defined(sun) && (defined(__sparc) || defined(__sparc__)) -#include /* for smt_pause(3c) */ -#endif /* if defined(sun) && (defined(__sparc) || defined(__sparc__)) */ - #include #include +#include #include #include #include @@ -42,29 +39,6 @@ #define RWLOCK_MAX_ADAPTIVE_COUNT 100 #endif /* ifndef RWLOCK_MAX_ADAPTIVE_COUNT */ -#if defined(_MSC_VER) -#include -#define isc_rwlock_pause() YieldProcessor() -#elif defined(__x86_64__) -#include -#define isc_rwlock_pause() _mm_pause() -#elif defined(__i386__) -#define isc_rwlock_pause() __asm__ __volatile__("rep; nop") -#elif defined(__ia64__) -#define isc_rwlock_pause() __asm__ __volatile__("hint @pause") -#elif defined(__arm__) && HAVE_ARM_YIELD -#define isc_rwlock_pause() __asm__ __volatile__("yield") -#elif defined(sun) && (defined(__sparc) || defined(__sparc__)) -#define isc_rwlock_pause() smt_pause() -#elif (defined(__sparc) || defined(__sparc__)) && HAVE_SPARC_PAUSE -#define isc_rwlock_pause() __asm__ __volatile__("pause") -#elif defined(__ppc__) || defined(_ARCH_PPC) || defined(_ARCH_PWR) || \ - defined(_ARCH_PWR2) || defined(_POWER) -#define isc_rwlock_pause() __asm__ volatile("or 27,27,27") -#else /* if defined(_MSC_VER) */ -#define isc_rwlock_pause() -#endif /* if defined(_MSC_VER) */ - #ifdef ISC_RWLOCK_TRACE #include /* Required for fprintf/stderr. */ @@ -331,7 +305,7 @@ isc__rwlock_lock(isc__rwlock_t *rwl, isc_rwlocktype_t type) { rwlock_lock(rwl, type); break; } - isc_rwlock_pause(); + isc_pause(); } while (isc_rwlock_trylock(rwl, type) != ISC_R_SUCCESS); atomic_fetch_add_release(&rwl->spins, (cnt - spins) / 8);