From cfedf924d36debd2ea4fe146ebe33f3623f12cf5 Mon Sep 17 00:00:00 2001 From: Attilio Rao Date: Sat, 3 Nov 2012 23:03:14 +0000 Subject: [PATCH] Rework the known rwlock to benefit about staying on their own cache line in order to avoid manual frobbing but using struct rwlock_padalign. Reviewed by: alc, jimharris --- sys/amd64/amd64/pmap.c | 11 +---------- sys/i386/i386/pmap.c | 11 +---------- sys/ia64/ia64/pmap.c | 11 +---------- sys/mips/mips/pmap.c | 11 +---------- sys/powerpc/aim/mmu_oea.c | 11 +---------- sys/sparc64/include/pmap.h | 8 +------- sys/sparc64/sparc64/pmap.c | 7 +------ 7 files changed, 7 insertions(+), 63 deletions(-) diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c index a80fc7b8f5f..06b45b208c1 100644 --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -225,16 +225,7 @@ u_int64_t KPML4phys; /* phys addr of kernel level 4 */ static u_int64_t DMPDphys; /* phys addr of direct mapped level 2 */ static u_int64_t DMPDPphys; /* phys addr of direct mapped level 3 */ -/* - * Isolate the global pv list lock from data and other locks to prevent false - * sharing within the cache. - */ -static struct { - struct rwlock lock; - char padding[CACHE_LINE_SIZE - sizeof(struct rwlock)]; -} pvh_global __aligned(CACHE_LINE_SIZE); - -#define pvh_global_lock pvh_global.lock +static struct rwlock_padalign pvh_global_lock; /* * Data for the pv entry allocation mechanism diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c index b9033da4f20..f14f2151a2c 100644 --- a/sys/i386/i386/pmap.c +++ b/sys/i386/i386/pmap.c @@ -224,16 +224,7 @@ SYSCTL_INT(_vm_pmap, OID_AUTO, pg_ps_enabled, CTLFLAG_RDTUN, &pg_ps_enabled, 0, #define PAT_INDEX_SIZE 8 static int pat_index[PAT_INDEX_SIZE]; /* cache mode to PAT index conversion */ -/* - * Isolate the global pv list lock from data and other locks to prevent false - * sharing within the cache. - */ -static struct { - struct rwlock lock; - char padding[CACHE_LINE_SIZE - sizeof(struct rwlock)]; -} pvh_global __aligned(CACHE_LINE_SIZE); - -#define pvh_global_lock pvh_global.lock +static struct rwlock_padalign pvh_global_lock; /* * Data for the pv entry allocation mechanism diff --git a/sys/ia64/ia64/pmap.c b/sys/ia64/ia64/pmap.c index f71781c3f8f..594f8c6830d 100644 --- a/sys/ia64/ia64/pmap.c +++ b/sys/ia64/ia64/pmap.c @@ -214,16 +214,7 @@ static int pmap_ridmax; static uint64_t *pmap_ridmap; struct mtx pmap_ridmutex; -/* - * Isolate the global pv list lock from data and other locks to prevent false - * sharing within the cache. - */ -static struct { - struct rwlock lock; - char padding[CACHE_LINE_SIZE - sizeof(struct rwlock)]; -} pvh_global __aligned(CACHE_LINE_SIZE); - -#define pvh_global_lock pvh_global.lock +static struct rwlock_padalign pvh_global_lock; /* * Data for the pv entry allocation mechanism diff --git a/sys/mips/mips/pmap.c b/sys/mips/mips/pmap.c index 7737757b6c6..608546e8413 100644 --- a/sys/mips/mips/pmap.c +++ b/sys/mips/mips/pmap.c @@ -148,16 +148,7 @@ vm_offset_t kernel_vm_end = VM_MIN_KERNEL_ADDRESS; static void pmap_asid_alloc(pmap_t pmap); -/* - * Isolate the global pv list lock from data and other locks to prevent false - * sharing within the cache. - */ -static struct { - struct rwlock lock; - char padding[CACHE_LINE_SIZE - sizeof(struct rwlock)]; -} pvh_global __aligned(CACHE_LINE_SIZE); - -#define pvh_global_lock pvh_global.lock +static struct rwlock_padalign pvh_global_lock; /* * Data for the pv entry allocation mechanism diff --git a/sys/powerpc/aim/mmu_oea.c b/sys/powerpc/aim/mmu_oea.c index a67bb9c17ff..3ed8eb1e0ac 100644 --- a/sys/powerpc/aim/mmu_oea.c +++ b/sys/powerpc/aim/mmu_oea.c @@ -200,16 +200,7 @@ struct pvo_head *moea_pvo_table; /* pvo entries by pteg index */ struct pvo_head moea_pvo_kunmanaged = LIST_HEAD_INITIALIZER(moea_pvo_kunmanaged); /* list of unmanaged pages */ -/* - * Isolate the global pv list lock from data and other locks to prevent false - * sharing within the cache. - */ -static struct { - struct rwlock lock; - char padding[CACHE_LINE_SIZE - sizeof(struct rwlock)]; -} pvh_global __aligned(CACHE_LINE_SIZE); - -#define pvh_global_lock pvh_global.lock +static struct rwlock_padalign pvh_global_lock; uma_zone_t moea_upvo_zone; /* zone for pvo entries for unmanaged pages */ uma_zone_t moea_mpvo_zone; /* zone for pvo entries for managed pages */ diff --git a/sys/sparc64/include/pmap.h b/sys/sparc64/include/pmap.h index cf2b2af12ef..6e2575d0190 100644 --- a/sys/sparc64/include/pmap.h +++ b/sys/sparc64/include/pmap.h @@ -68,11 +68,6 @@ struct pmap { struct pmap_statistics pm_stats; }; -struct tte_list_lock { - struct rwlock lock; - char padding[CACHE_LINE_SIZE - sizeof(struct rwlock)]; -}; - #define PMAP_LOCK(pmap) mtx_lock(&(pmap)->pm_mtx) #define PMAP_LOCK_ASSERT(pmap, type) \ mtx_assert(&(pmap)->pm_mtx, (type)) @@ -108,8 +103,7 @@ void pmap_set_kctx(void); extern struct pmap kernel_pmap_store; #define kernel_pmap (&kernel_pmap_store) -extern struct tte_list_lock tte_list_global; -#define tte_list_global_lock tte_list_global.lock +extern struct rwlock_padalign tte_list_global_lock; extern vm_paddr_t phys_avail[]; extern vm_offset_t virtual_avail; extern vm_offset_t virtual_end; diff --git a/sys/sparc64/sparc64/pmap.c b/sys/sparc64/sparc64/pmap.c index 0e1c8f9a022..d20d69fde1c 100644 --- a/sys/sparc64/sparc64/pmap.c +++ b/sys/sparc64/sparc64/pmap.c @@ -129,12 +129,7 @@ vm_offset_t vm_max_kernel_address; */ struct pmap kernel_pmap_store; -/* - * Isolate the global TTE list lock from data and other locks to prevent - * false sharing within the cache (see also the declaration of struct - * tte_list_lock). - */ -struct tte_list_lock tte_list_global __aligned(CACHE_LINE_SIZE); +struct rwlock_padalign tte_list_global_lock; /* * Allocate physical memory for use in pmap_bootstrap.