diff --git a/CHANGES b/CHANGES index 81a037d3cc..0325dce907 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,7 @@ +3440. [performance] Implement adaptive read-write locks, reducing the + overhead of locks that are only held briefly. + [RT #37329] + 4339. [test] Use "mdig" to test pipelined queries. [RT #41929] 4338. [bug] Reimplement change 4324 as it wasn't properly doing diff --git a/configure b/configure index 5483289f0f..7ce158192d 100755 --- a/configure +++ b/configure @@ -712,6 +712,7 @@ DNSTAPOBJS DNSTAPSRCS DNSTAP PROTOC_C +ISC_PLATFORM_BUSYWAITNOP ISC_ARCH_DIR ISC_PLATFORM_USEMACASM ISC_PLATFORM_USESTDASM @@ -19964,6 +19965,115 @@ $as_echo "#define HAVE_BUILTIN_EXPECT 1" >>confdefs.h fi +# +# CPU relax (for spin locks) +# +if $use_threads +then + case "$host" in + i[3456]86-*) + # x86_32 + { $as_echo "$as_me:${as_lineno-$LINENO}: checking if asm(\"rep; nop\"); works" >&5 +$as_echo_n "checking if asm(\"rep; nop\"); works... " >&6; } + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ +asm("rep; nop"); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + ISC_PLATFORM_BUSYWAITNOP="#define ISC_PLATFORM_BUSYWAITNOP asm(\"rep; nop\")" +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + ;; + x86_64-*|amd64-*) + # x86_64 + { $as_echo "$as_me:${as_lineno-$LINENO}: checking if asm(\"rep; nop\"); works" >&5 +$as_echo_n "checking if asm(\"rep; nop\"); works... " >&6; } + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ +asm("rep; nop"); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + ISC_PLATFORM_BUSYWAITNOP="#define ISC_PLATFORM_BUSYWAITNOP asm(\"rep; nop\")" +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + ;; + ia64-*) + # ia64 + { $as_echo "$as_me:${as_lineno-$LINENO}: checking if asm(\"hint @pause\"); works" >&5 +$as_echo_n "checking if asm(\"hint @pause\"); works... " >&6; } + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ +asm("hint @pause"); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + ISC_PLATFORM_BUSYWAITNOP="#define ISC_PLATFORM_BUSYWAITNOP asm(\"hint @pause\")" +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + ;; + sparc-*) + # sparc + { $as_echo "$as_me:${as_lineno-$LINENO}: checking if cpu_relax(); or __cpu_relax(); works" >&5 +$as_echo_n "checking if cpu_relax(); or __cpu_relax(); works... " >&6; } + ac_fn_c_check_func "$LINENO" "cpu_relax" "ac_cv_func_cpu_relax" +if test "x$ac_cv_func_cpu_relax" = xyes; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + ISC_PLATFORM_BUSYWAITNOP="#define ISC_PLATFORM_BUSYWAITNOP cpu_relax()" +else + ac_fn_c_check_func "$LINENO" "__cpu_relax" "ac_cv_func___cpu_relax" +if test "x$ac_cv_func___cpu_relax" = xyes; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + ISC_PLATFORM_BUSYWAITNOP="#define ISC_PLATFORM_BUSYWAITNOP __cpu_relax()" +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + +fi + + ;; + esac +fi + + + # # Activate "rrset-order fixed" or not? # diff --git a/configure.in b/configure.in index 506a49e58a..e6a8b3e864 100644 --- a/configure.in +++ b/configure.in @@ -3910,6 +3910,58 @@ if test "$have_builtin_expect" = "yes"; then AC_DEFINE(HAVE_BUILTIN_EXPECT, 1, [Define to 1 if the compiler supports __builtin_expect.]) fi +# +# CPU relax (for spin locks) +# +if $use_threads +then + case "$host" in + [i[3456]86-*]) + # x86_32 + AC_MSG_CHECKING([if asm("rep; nop"); works]) + AC_TRY_COMPILE(,[asm("rep; nop");], + [AC_MSG_RESULT(yes) + ISC_PLATFORM_BUSYWAITNOP="#define ISC_PLATFORM_BUSYWAITNOP asm(\"rep; nop\")"], + [AC_MSG_RESULT(no)], + [AC_MSG_RESULT([cross compile, assume yes]) + ISC_PLATFORM_BUSYWAITNOP="#define ISC_PLATFORM_BUSYWAITNOP asm(\"rep; nop\")"]) + ;; + x86_64-*|amd64-*) + # x86_64 + AC_MSG_CHECKING([if asm("rep; nop"); works]) + AC_TRY_COMPILE(,[asm("rep; nop");], + [AC_MSG_RESULT(yes) + ISC_PLATFORM_BUSYWAITNOP="#define ISC_PLATFORM_BUSYWAITNOP asm(\"rep; nop\")"], + [AC_MSG_RESULT(no)], + [AC_MSG_RESULT([cross compile, assume yes]) + ISC_PLATFORM_BUSYWAITNOP="#define ISC_PLATFORM_BUSYWAITNOP asm(\"rep; nop\")"]) + ;; + ia64-*) + # ia64 + AC_MSG_CHECKING([if asm("hint @pause"); works]) + AC_TRY_COMPILE(,[asm("hint @pause");], + [AC_MSG_RESULT(yes) + ISC_PLATFORM_BUSYWAITNOP="#define ISC_PLATFORM_BUSYWAITNOP asm(\"hint @pause\")"], + [AC_MSG_RESULT(no)], + [AC_MSG_RESULT([cross compile, assume yes]) + ISC_PLATFORM_BUSYWAITNOP="#define ISC_PLATFORM_BUSYWAITNOP asm(\"hint @pause\")"]) + ;; + sparc-*) + # sparc + AC_MSG_CHECKING([if cpu_relax(); or __cpu_relax(); works]) + AC_CHECK_FUNC(cpu_relax, + [AC_MSG_RESULT(yes) + ISC_PLATFORM_BUSYWAITNOP="#define ISC_PLATFORM_BUSYWAITNOP cpu_relax()"], + [AC_CHECK_FUNC(__cpu_relax, + [AC_MSG_RESULT(yes) + ISC_PLATFORM_BUSYWAITNOP="#define ISC_PLATFORM_BUSYWAITNOP __cpu_relax()"], + [AC_MSG_RESULT(no)])]) + ;; + esac +fi + +AC_SUBST(ISC_PLATFORM_BUSYWAITNOP) + # # Activate "rrset-order fixed" or not? # diff --git a/lib/isc/include/isc/platform.h.in b/lib/isc/include/isc/platform.h.in index 75cdec33a3..2b451409e6 100644 --- a/lib/isc/include/isc/platform.h.in +++ b/lib/isc/include/isc/platform.h.in @@ -320,6 +320,11 @@ */ @ISC_PLATFORM_USESTDASM@ +/* + * Define with the busy wait nop asm or function call. + */ +@ISC_PLATFORM_BUSYWAITNOP@ + /* * Define if the platform has . */ diff --git a/lib/isc/include/isc/rwlock.h b/lib/isc/include/isc/rwlock.h index 28052cdd7f..9186c8afc6 100644 --- a/lib/isc/include/isc/rwlock.h +++ b/lib/isc/include/isc/rwlock.h @@ -44,6 +44,7 @@ struct isc_rwlock { /* Unlocked. */ unsigned int magic; isc_mutex_t lock; + isc_int32_t spins; #if defined(ISC_PLATFORM_HAVEXADD) && defined(ISC_PLATFORM_HAVECMPXCHG) /* diff --git a/lib/isc/rwlock.c b/lib/isc/rwlock.c index 461d6e295a..6e29008865 100644 --- a/lib/isc/rwlock.c +++ b/lib/isc/rwlock.c @@ -44,6 +44,15 @@ #define RWLOCK_DEFAULT_WRITE_QUOTA 4 #endif +#ifndef RWLOCK_MAX_ADAPTIVE_COUNT +#define RWLOCK_MAX_ADAPTIVE_COUNT 100 +#endif + +#if defined(ISC_PLATFORM_HAVEXADD) && defined(ISC_PLATFORM_HAVECMPXCHG) +static isc_result_t +isc__rwlock_lock(isc_rwlock_t *rwl, isc_rwlocktype_t type); +#endif + #ifdef ISC_RWLOCK_TRACE #include /* Required for fprintf/stderr. */ #include /* Required for isc_thread_self(). */ @@ -85,6 +94,7 @@ isc_rwlock_init(isc_rwlock_t *rwl, unsigned int read_quota, */ rwl->magic = 0; + rwl->spins = 0; #if defined(ISC_PLATFORM_HAVEXADD) && defined(ISC_PLATFORM_HAVECMPXCHG) rwl->write_requests = 0; rwl->write_completions = 0; @@ -238,8 +248,8 @@ isc_rwlock_destroy(isc_rwlock_t *rwl) { #define WRITER_ACTIVE 0x1 #define READER_INCR 0x2 -isc_result_t -isc_rwlock_lock(isc_rwlock_t *rwl, isc_rwlocktype_t type) { +static isc_result_t +isc__rwlock_lock(isc_rwlock_t *rwl, isc_rwlocktype_t type) { isc_int32_t cntflag; REQUIRE(VALID_RWLOCK(rwl)); @@ -348,6 +358,30 @@ isc_rwlock_lock(isc_rwlock_t *rwl, isc_rwlocktype_t type) { return (ISC_R_SUCCESS); } +isc_result_t +isc_rwlock_lock(isc_rwlock_t *rwl, isc_rwlocktype_t type) { + isc_int32_t cnt = 0; + isc_int32_t max_cnt = rwl->spins * 2 + 10; + isc_result_t result = ISC_R_SUCCESS; + + if (max_cnt > RWLOCK_MAX_ADAPTIVE_COUNT) + max_cnt = RWLOCK_MAX_ADAPTIVE_COUNT; + + do { + if (cnt++ >= max_cnt) { + result = isc__rwlock_lock(rwl, type); + break; + } +#ifdef ISC_PLATFORM_BUSYWAITNOP + ISC_PLATFORM_BUSYWAITNOP; +#endif + } while (isc_rwlock_trylock(rwl, type) != ISC_R_SUCCESS); + + rwl->spins += (cnt - rwl->spins) / 8; + + return (result); +} + isc_result_t isc_rwlock_trylock(isc_rwlock_t *rwl, isc_rwlocktype_t type) { isc_int32_t cntflag; @@ -606,7 +640,26 @@ doit(isc_rwlock_t *rwl, isc_rwlocktype_t type, isc_boolean_t nonblock) { isc_result_t isc_rwlock_lock(isc_rwlock_t *rwl, isc_rwlocktype_t type) { - return (doit(rwl, type, ISC_FALSE)); + isc_int32_t cnt = 0; + isc_int32_t max_cnt = rwl->spins * 2 + 10; + isc_result_t result = ISC_R_SUCCESS; + + if (max_cnt > RWLOCK_MAX_ADAPTIVE_COUNT) + max_cnt = RWLOCK_MAX_ADAPTIVE_COUNT; + + do { + if (cnt++ >= max_cnt) { + result = doit(rwl, type, ISC_FALSE); + break; + } +#ifdef ISC_PLATFORM_BUSYWAITNOP + ISC_PLATFORM_BUSYWAITNOP; +#endif + } while (doit(rwl, type, ISC_TRUE) != ISC_R_SUCCESS); + + rwl->spins += (cnt - rwl->spins) / 8; + + return (ISC_R_SUCCESS); } isc_result_t diff --git a/lib/isc/win32/include/isc/platform.h.in b/lib/isc/win32/include/isc/platform.h.in index bbd35aca93..12a2b4f78c 100644 --- a/lib/isc/win32/include/isc/platform.h.in +++ b/lib/isc/win32/include/isc/platform.h.in @@ -109,6 +109,11 @@ */ @ISC_PLATFORM_HAVECMPXCHG@ +/* + * Define with the busy wait nop asm or function call. + */ +@ISC_PLATFORM_BUSYWAITNOP@ + /* * If the strcasestr() operation is not available on this platform, * ISC_PLATFORM_NEEDSTRCASESTR will be defined. diff --git a/win32utils/Configure b/win32utils/Configure index 28899ef20f..79ae31e31e 100644 --- a/win32utils/Configure +++ b/win32utils/Configure @@ -383,7 +383,8 @@ my @substdefh = ("AES_CC", my %configdefp; -my @substdefp = ("ISC_PLATFORM_HAVEATOMICSTORE", +my @substdefp = ("ISC_PLATFORM_BUSYWAITNOP", + "ISC_PLATFORM_HAVEATOMICSTORE", "ISC_PLATFORM_HAVEATOMICSTOREQ", "ISC_PLATFORM_HAVECMPXCHG", "ISC_PLATFORM_HAVEXADD", @@ -693,11 +694,13 @@ if (($want_win32 eq "yes") && ($want_x64 eq "yes")) { $configvar{"BUILD_PLATFORM"} = "Win32"; $configvar{"MACHINE"} = "/machine:X86"; $configvar{"BUILD_MACHINE"} = "/machine:X86"; + $configdefp{"ISC_PLATFORM_BUSYWAITNOP"} = "__asm { rep nop }"; } elsif ($want_x64 eq "yes") { $configvar{"PLATFORM"} = "x64"; $configvar{"BUILD_PLATFORM"} = "x64"; $configvar{"MACHINE"} = "/machine:X64"; $configvar{"BUILD_MACHINE"} = "/machine:X64"; + $configdefp{"ISC_PLATFORM_BUSYWAITNOP"} = "__asm { rep nop }"; } # get the version information