diff --git a/lib/libthr/thread/thr_init.c b/lib/libthr/thread/thr_init.c index 981941b1d9d..d35834c409b 100644 --- a/lib/libthr/thread/thr_init.c +++ b/lib/libthr/thread/thr_init.c @@ -103,6 +103,8 @@ size_t _thr_guard_default; size_t _thr_stack_default = THR_STACK_DEFAULT; size_t _thr_stack_initial = THR_STACK_INITIAL; int _thr_page_size; +int _thr_spinloops; +int _thr_yieldloops; int _gc_count; struct umutex _mutex_static_lock = DEFAULT_UMUTEX; struct umutex _cond_static_lock = DEFAULT_UMUTEX; @@ -423,6 +425,7 @@ init_private(void) { size_t len; int mib[2]; + char *env; _thr_umutex_init(&_mutex_static_lock); _thr_umutex_init(&_cond_static_lock); @@ -452,7 +455,12 @@ init_private(void) _thr_guard_default = _thr_page_size; _pthread_attr_default.guardsize_attr = _thr_guard_default; _pthread_attr_default.stacksize_attr = _thr_stack_default; - + env = getenv("LIBPTHREAD_SPINLOOPS"); + if (env) + _thr_spinloops = atoi(env); + env = getenv("LIBPTHREAD_YIELDLOOPS"); + if (env) + _thr_yieldloops = atoi(env); TAILQ_INIT(&_thr_atfork_list); } init_once = 1; diff --git a/lib/libthr/thread/thr_mutex.c b/lib/libthr/thread/thr_mutex.c index b1e6d0c770c..2cc3ce0d075 100644 --- a/lib/libthr/thread/thr_mutex.c +++ b/lib/libthr/thread/thr_mutex.c @@ -66,12 +66,6 @@ #define MUTEX_ASSERT_NOT_OWNED(m) #endif -/* - * For adaptive mutexes, how many times to spin doing trylock2 - * before entering the kernel to block - */ -#define MUTEX_ADAPTIVE_SPINS 200 - /* * Prototypes */ @@ -279,6 +273,16 @@ _pthread_mutex_destroy(pthread_mutex_t *mutex) return (ret); } + +#define ENQUEUE_MUTEX(curthread, m) \ + m->m_owner = curthread; \ + /* Add to the list of owned mutexes: */ \ + MUTEX_ASSERT_NOT_OWNED(m); \ + if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0) \ + TAILQ_INSERT_TAIL(&curthread->mutexq, m, m_qe); \ + else \ + TAILQ_INSERT_TAIL(&curthread->pp_mutexq, m, m_qe) + static int mutex_trylock_common(struct pthread *curthread, pthread_mutex_t *mutex) { @@ -290,13 +294,7 @@ mutex_trylock_common(struct pthread *curthread, pthread_mutex_t *mutex) m = *mutex; ret = _thr_umutex_trylock(&m->m_lock, id); if (ret == 0) { - m->m_owner = curthread; - /* Add to the list of owned mutexes. */ - MUTEX_ASSERT_NOT_OWNED(m); - if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0) - TAILQ_INSERT_TAIL(&curthread->mutexq, m, m_qe); - else - TAILQ_INSERT_TAIL(&curthread->pp_mutexq, m, m_qe); + ENQUEUE_MUTEX(curthread, m); } else if (m->m_owner == curthread) { ret = mutex_self_trylock(m); } /* else {} */ @@ -348,39 +346,43 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *mutex, struct pthread_mutex *m; uint32_t id; int ret; + int count; id = TID(curthread); m = *mutex; ret = _thr_umutex_trylock2(&m->m_lock, id); if (ret == 0) { - m->m_owner = curthread; - /* Add to the list of owned mutexes: */ - MUTEX_ASSERT_NOT_OWNED(m); - if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0) - TAILQ_INSERT_TAIL(&curthread->mutexq, m, m_qe); - else - TAILQ_INSERT_TAIL(&curthread->pp_mutexq, m, m_qe); + ENQUEUE_MUTEX(curthread, m); } else if (m->m_owner == curthread) { ret = mutex_self_lock(m, abstime); } else { - /* - * For adaptive mutexes, spin for a bit in the expectation - * that if the application requests this mutex type then - * the lock is likely to be released quickly and it is - * faster than entering the kernel - */ - if (m->m_type == PTHREAD_MUTEX_ADAPTIVE_NP) { - int count = MUTEX_ADAPTIVE_SPINS; - - while (count--) { + if (_thr_spinloops != 0 && _thr_is_smp && + !(m->m_lock.m_flags & UMUTEX_PRIO_PROTECT)) { + count = _thr_spinloops; + while (count && m->m_lock.m_owner != UMUTEX_UNOWNED) { + count--; + CPU_SPINWAIT; + } + if (count) { ret = _thr_umutex_trylock2(&m->m_lock, id); - if (ret == 0) - break; - cpu_spinwait(); + if (ret == 0) { + ENQUEUE_MUTEX(curthread, m); + return (ret); + } + } + } + + if (_thr_yieldloops != 0) { + count = _thr_yieldloops; + while (count--) { + _sched_yield(); + ret = _thr_umutex_trylock2(&m->m_lock, id); + if (ret == 0) { + ENQUEUE_MUTEX(curthread, m); + return (ret); + } } } - if (ret == 0) - goto done; if (abstime == NULL) { ret = __thr_umutex_lock(&m->m_lock); @@ -399,17 +401,8 @@ mutex_lock_common(struct pthread *curthread, pthread_mutex_t *mutex, if (ret == EINTR) ret = ETIMEDOUT; } -done: - if (ret == 0) { - m->m_owner = curthread; - /* Add to the list of owned mutexes: */ - MUTEX_ASSERT_NOT_OWNED(m); - if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0) - TAILQ_INSERT_TAIL(&curthread->mutexq, m, m_qe); - else - TAILQ_INSERT_TAIL(&curthread->pp_mutexq, m, - m_qe); - } + if (ret == 0) + ENQUEUE_MUTEX(curthread, m); } return (ret); } @@ -529,7 +522,6 @@ mutex_self_trylock(pthread_mutex_t m) switch (m->m_type) { case PTHREAD_MUTEX_ERRORCHECK: case PTHREAD_MUTEX_NORMAL: - case PTHREAD_MUTEX_ADAPTIVE_NP: ret = EBUSY; break; diff --git a/lib/libthr/thread/thr_private.h b/lib/libthr/thread/thr_private.h index ab64bf7a6c7..313074f20bd 100644 --- a/lib/libthr/thread/thr_private.h +++ b/lib/libthr/thread/thr_private.h @@ -583,7 +583,9 @@ extern size_t _thr_guard_default __hidden; extern size_t _thr_stack_default __hidden; extern size_t _thr_stack_initial __hidden; extern int _thr_page_size __hidden; -extern int _thr_adaptive_spin __hidden; +extern int _thr_spinloops __hidden; +extern int _thr_yieldloops __hidden; + /* Garbage thread count. */ extern int _gc_count __hidden; @@ -653,6 +655,7 @@ int _schedparam_to_rtp(int policy, const struct sched_param *param, struct rtprio *rtp) __hidden; void _thread_bp_create(void); void _thread_bp_death(void); +int _sched_yield(void); /* #include */ #ifdef _SYS_FCNTL_H_