From fc4f686d596c85f707eaec46b78ed03f2ba81515 Mon Sep 17 00:00:00 2001 From: Mateusz Guzik Date: Wed, 1 Jun 2016 18:32:20 +0000 Subject: [PATCH] Microoptimize locking primitives by avoiding unnecessary atomic ops. Inline version of primitives do an atomic op and if it fails they fallback to actual primitives, which immediately retry the atomic op. The obvious optimisation is to check if the lock is free and only then proceed to do an atomic op. Reviewed by: jhb, vangyzen --- sys/kern/kern_lock.c | 12 +++++++++--- sys/kern/kern_mutex.c | 13 +++++++++---- sys/kern/kern_rwlock.c | 4 +++- sys/kern/kern_sx.c | 5 ++++- sys/sys/mutex.h | 6 +++--- sys/sys/rwlock.h | 4 ++-- sys/sys/sx.h | 6 ++++-- 7 files changed, 34 insertions(+), 16 deletions(-) diff --git a/sys/kern/kern_lock.c b/sys/kern/kern_lock.c index 34d7b34b618..1b10623519f 100644 --- a/sys/kern/kern_lock.c +++ b/sys/kern/kern_lock.c @@ -787,8 +787,10 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk, break; } - while (!atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, - tid)) { + for (;;) { + if (lk->lk_lock == LK_UNLOCKED && + atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) + break; #ifdef HWPMC_HOOKS PMC_SOFT_CALL( , , lock, failed); #endif @@ -1124,7 +1126,11 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk, __func__, iwmesg, file, line); } - while (!atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) { + for (;;) { + if (lk->lk_lock == LK_UNLOCKED && + atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) + break; + #ifdef HWPMC_HOOKS PMC_SOFT_CALL( , , lock, failed); #endif diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c index 3f62d171528..d167205dd52 100644 --- a/sys/kern/kern_mutex.c +++ b/sys/kern/kern_mutex.c @@ -419,7 +419,9 @@ __mtx_lock_sleep(volatile uintptr_t *c, uintptr_t tid, int opts, all_time -= lockstat_nsecs(&m->lock_object); #endif - while (!_mtx_obtain_lock(m, tid)) { + for (;;) { + if (m->mtx_lock == MTX_UNOWNED && _mtx_obtain_lock(m, tid)) + break; #ifdef KDTRACE_HOOKS spin_cnt++; #endif @@ -602,8 +604,9 @@ _mtx_lock_spin_cookie(volatile uintptr_t *c, uintptr_t tid, int opts, #ifdef KDTRACE_HOOKS spin_time -= lockstat_nsecs(&m->lock_object); #endif - while (!_mtx_obtain_lock(m, tid)) { - + for (;;) { + if (m->mtx_lock == MTX_UNOWNED && _mtx_obtain_lock(m, tid)) + break; /* Give interrupts a chance while we spin. */ spinlock_exit(); while (m->mtx_lock != MTX_UNOWNED) { @@ -675,7 +678,9 @@ retry: m->lock_object.lo_name, file, line)); WITNESS_CHECKORDER(&m->lock_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL); - while (!_mtx_obtain_lock(m, tid)) { + for (;;) { + if (m->mtx_lock == MTX_UNOWNED && _mtx_obtain_lock(m, tid)) + break; if (m->mtx_lock == tid) { m->mtx_recurse++; break; diff --git a/sys/kern/kern_rwlock.c b/sys/kern/kern_rwlock.c index 6541724f5ae..6a904d2a640 100644 --- a/sys/kern/kern_rwlock.c +++ b/sys/kern/kern_rwlock.c @@ -771,7 +771,9 @@ __rw_wlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file, all_time -= lockstat_nsecs(&rw->lock_object); state = rw->rw_lock; #endif - while (!_rw_write_lock(rw, tid)) { + for (;;) { + if (rw->rw_lock == RW_UNLOCKED && _rw_write_lock(rw, tid)) + break; #ifdef KDTRACE_HOOKS spin_cnt++; #endif diff --git a/sys/kern/kern_sx.c b/sys/kern/kern_sx.c index 96e117b429b..2a81c04d8c7 100644 --- a/sys/kern/kern_sx.c +++ b/sys/kern/kern_sx.c @@ -544,7 +544,10 @@ _sx_xlock_hard(struct sx *sx, uintptr_t tid, int opts, const char *file, all_time -= lockstat_nsecs(&sx->lock_object); state = sx->sx_lock; #endif - while (!atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid)) { + for (;;) { + if (sx->sx_lock == SX_LOCK_UNLOCKED && + atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid)) + break; #ifdef KDTRACE_HOOKS spin_cnt++; #endif diff --git a/sys/sys/mutex.h b/sys/sys/mutex.h index a9ec072f8d7..0443922b445 100644 --- a/sys/sys/mutex.h +++ b/sys/sys/mutex.h @@ -185,7 +185,7 @@ void thread_lock_flags_(struct thread *, int, const char *, int); #define __mtx_lock(mp, tid, opts, file, line) do { \ uintptr_t _tid = (uintptr_t)(tid); \ \ - if (!_mtx_obtain_lock((mp), _tid)) \ + if (((mp)->mtx_lock != MTX_UNOWNED || !_mtx_obtain_lock((mp), _tid)))\ _mtx_lock_sleep((mp), _tid, (opts), (file), (line)); \ else \ LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(adaptive__acquire, \ @@ -203,7 +203,7 @@ void thread_lock_flags_(struct thread *, int, const char *, int); uintptr_t _tid = (uintptr_t)(tid); \ \ spinlock_enter(); \ - if (!_mtx_obtain_lock((mp), _tid)) { \ + if (((mp)->mtx_lock != MTX_UNOWNED || !_mtx_obtain_lock((mp), _tid))) {\ if ((mp)->mtx_lock == _tid) \ (mp)->mtx_recurse++; \ else \ @@ -232,7 +232,7 @@ void thread_lock_flags_(struct thread *, int, const char *, int); \ if ((mp)->mtx_recurse == 0) \ LOCKSTAT_PROFILE_RELEASE_LOCK(adaptive__release, mp); \ - if (!_mtx_release_lock((mp), _tid)) \ + if ((mp)->mtx_lock != _tid || !_mtx_release_lock((mp), _tid)) \ _mtx_unlock_sleep((mp), (opts), (file), (line)); \ } while (0) diff --git a/sys/sys/rwlock.h b/sys/sys/rwlock.h index f8947c55617..6b4f505e0d0 100644 --- a/sys/sys/rwlock.h +++ b/sys/sys/rwlock.h @@ -96,7 +96,7 @@ #define __rw_wlock(rw, tid, file, line) do { \ uintptr_t _tid = (uintptr_t)(tid); \ \ - if (!_rw_write_lock((rw), _tid)) \ + if ((rw)->rw_lock != RW_UNLOCKED || !_rw_write_lock((rw), _tid))\ _rw_wlock_hard((rw), _tid, (file), (line)); \ else \ LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw, \ @@ -112,7 +112,7 @@ else { \ LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw, \ LOCKSTAT_WRITER); \ - if (!_rw_write_unlock((rw), _tid)) \ + if ((rw)->rw_lock != _tid || !_rw_write_unlock((rw), _tid))\ _rw_wunlock_hard((rw), _tid, (file), (line)); \ } \ } while (0) diff --git a/sys/sys/sx.h b/sys/sys/sx.h index 03b51d37803..57a31d9aace 100644 --- a/sys/sys/sx.h +++ b/sys/sys/sx.h @@ -150,7 +150,8 @@ __sx_xlock(struct sx *sx, struct thread *td, int opts, const char *file, uintptr_t tid = (uintptr_t)td; int error = 0; - if (!atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid)) + if (sx->sx_lock != SX_LOCK_UNLOCKED || + !atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid)) error = _sx_xlock_hard(sx, tid, opts, file, line); else LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire, sx, @@ -168,7 +169,8 @@ __sx_xunlock(struct sx *sx, struct thread *td, const char *file, int line) if (sx->sx_recurse == 0) LOCKSTAT_PROFILE_RELEASE_RWLOCK(sx__release, sx, LOCKSTAT_WRITER); - if (!atomic_cmpset_rel_ptr(&sx->sx_lock, tid, SX_LOCK_UNLOCKED)) + if (sx->sx_lock != tid || + !atomic_cmpset_rel_ptr(&sx->sx_lock, tid, SX_LOCK_UNLOCKED)) _sx_xunlock_hard(sx, tid, file, line); }