From 7e55df27cb533f87630cbf473f15efbb11e9321f Mon Sep 17 00:00:00 2001
From: Nathan Whitehorn <nwhitehorn@FreeBSD.org>
Date: Wed, 28 Mar 2012 17:25:29 +0000
Subject: [PATCH] More PMAP performance improvements: skip 256 MB segments
 entirely if they are are not mapped during ranged operations and reduce the
 scope of the tlbie lock only to the actual tlbie instruction instead of the
 entire sequence. There are a few more optimization possibilities here as
 well.

---
 sys/powerpc/aim/mmu_oea64.c     | 24 ++++++++++++++++++++----
 sys/powerpc/aim/moea64_native.c | 13 ++++++-------
 2 files changed, 26 insertions(+), 11 deletions(-)

diff --git a/sys/powerpc/aim/mmu_oea64.c b/sys/powerpc/aim/mmu_oea64.c
index 794f8910130..41361b7e81d 100644
--- a/sys/powerpc/aim/mmu_oea64.c
+++ b/sys/powerpc/aim/mmu_oea64.c
@@ -1981,10 +1981,18 @@ moea64_protect(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva,
 	LOCK_TABLE_RD();
 	PMAP_LOCK(pm);
 	if ((eva - sva)/PAGE_SIZE < pm->pm_stats.resident_count) {
-		for (; sva < eva; sva += PAGE_SIZE) {
+		while (sva < eva) {
+			#ifdef __powerpc64__
+			if (pm != kernel_pmap &&
+			    user_va_to_slb_entry(pm, sva) == NULL) {
+				sva = roundup2(sva + 1, SEGMENT_LENGTH);
+				continue;
+			}
+			#endif
 			pvo = moea64_pvo_find_va(pm, sva);
 			if (pvo != NULL)
 				moea64_pvo_protect(mmu, pm, pvo, prot);
+			sva += PAGE_SIZE;
 		}
 	} else {
 		LIST_FOREACH_SAFE(pvo, &pm->pmap_pvo, pvo_plink, tpvo) {
@@ -2095,10 +2103,18 @@ moea64_remove(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva)
 	LOCK_TABLE_WR();
 	PMAP_LOCK(pm);
 	if ((eva - sva)/PAGE_SIZE < pm->pm_stats.resident_count) {
-		for (; sva < eva; sva += PAGE_SIZE) {
+		while (sva < eva) {
+			#ifdef __powerpc64__
+			if (pm != kernel_pmap &&
+			    user_va_to_slb_entry(pm, sva) == NULL) {
+				sva = roundup2(sva + 1, SEGMENT_LENGTH);
+				continue;
+			}
+			#endif
 			pvo = moea64_pvo_find_va(pm, sva);
 			if (pvo != NULL)
 				moea64_pvo_remove(mmu, pvo);
+			sva += PAGE_SIZE;
 		}
 	} else {
 		LIST_FOREACH_SAFE(pvo, &pm->pmap_pvo, pvo_plink, tpvo) {
@@ -2566,7 +2582,7 @@ moea64_mapdev_attr(mmu_t mmu, vm_offset_t pa, vm_size_t size, vm_memattr_t ma)
 
 	ppa = trunc_page(pa);
 	offset = pa & PAGE_MASK;
-	size = roundup(offset + size, PAGE_SIZE);
+	size = roundup2(offset + size, PAGE_SIZE);
 
 	va = kmem_alloc_nofault(kernel_map, size);
 
@@ -2597,7 +2613,7 @@ moea64_unmapdev(mmu_t mmu, vm_offset_t va, vm_size_t size)
 
 	base = trunc_page(va);
 	offset = va & PAGE_MASK;
-	size = roundup(offset + size, PAGE_SIZE);
+	size = roundup2(offset + size, PAGE_SIZE);
 
 	kmem_free(kernel_map, base, size);
 }
diff --git a/sys/powerpc/aim/moea64_native.c b/sys/powerpc/aim/moea64_native.c
index 97766da84f5..081194f9870 100644
--- a/sys/powerpc/aim/moea64_native.c
+++ b/sys/powerpc/aim/moea64_native.c
@@ -103,6 +103,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/proc.h>
+#include <sys/sched.h>
 #include <sys/sysctl.h>
 #include <sys/systm.h>
 
@@ -152,15 +153,13 @@ TLBIE(uint64_t vpn) {
 	vpn &= ~(0xffffULL << 48);
 
 #ifdef __powerpc64__
+	sched_pin();
+	__asm __volatile("ptesync");
 	mtx_lock(&tlbie_mutex);
-	__asm __volatile("\
-	    ptesync; \
-	    tlbie %0; \
-	    eieio; \
-	    tlbsync; \
-	    ptesync;" 
-	:: "r"(vpn) : "memory");
+	__asm __volatile("tlbie %0" :: "r"(vpn) : "memory");
 	mtx_unlock(&tlbie_mutex);
+	__asm __volatile("eieio; tlbsync; ptesync");
+	sched_unpin();
 #else
 	vpn_hi = (uint32_t)(vpn >> 32);
 	vpn_lo = (uint32_t)vpn;