From 9d1b7fa31f5101f4a5c9e522f39e80945c34c9a1 Mon Sep 17 00:00:00 2001
From: Alan Cox <alc@FreeBSD.org>
Date: Thu, 12 Jun 2008 05:18:09 +0000
Subject: [PATCH] Reverse the direction of pmap_promote_pde()'s traversal over
 the specified page table page.  The direction of the traversal can matter if
 pmap_promote_pde() has to remove write access (PG_RW) from a PTE that hasn't
 been modified (PG_M).  In general, if there are two or more such PTEs to
 choose among, it is better to write protect the one nearer the high end of
 the page table page rather than the low end.  This is because most programs
 access memory in an ascending direction.  The net result of this change is a
 sometimes significant reduction in the number of failed promotion attempts
 and the number of pages that are write protected by pmap_promote_pde().

---
 sys/amd64/amd64/pmap.c | 38 +++++++++++++++++++++++++++-----------
 1 file changed, 27 insertions(+), 11 deletions(-)

diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
index 0f2f0c156c9..8114bb89d20 100644
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -2759,8 +2759,8 @@ retry:
 
 /*
  * Tries to promote the 512, contiguous 4KB page mappings that are within a
- * single page table page to a single 2MB page mapping.  For promotion to
- * occur, two conditions must be met: (1) the 4KB page mappings must map
+ * single page table page (PTP) to a single 2MB page mapping.  For promotion
+ * to occur, two conditions must be met: (1) the 4KB page mappings must map
  * aligned, contiguous physical memory and (2) the 4KB page mappings must have
  * identical characteristics. 
  */
@@ -2774,23 +2774,39 @@ pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)
 	vm_page_t mpte;
 
 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+
+	/*
+	 * Examine the first PTE in the specified PTP.  Abort if this PTE is
+	 * either invalid, unused, or does not map the first 4KB physical page
+	 * within a 2MB page. 
+	 */
 	firstpte = (pt_entry_t *)PHYS_TO_DMAP(*pde & PG_FRAME);
+setpde:
 	newpde = *firstpte;
-	if ((newpde & (PG_A | PG_V)) != (PG_A | PG_V)) {
+	if ((newpde & ((PG_FRAME & PDRMASK) | PG_A | PG_V)) != (PG_A | PG_V)) {
 		pmap_pde_p_failures++;
 		CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx"
 		    " in pmap %p", va, pmap);
 		return;
 	}
-	if ((newpde & (PG_M | PG_RW)) == PG_RW)
+	if ((newpde & (PG_M | PG_RW)) == PG_RW) {
+		/*
+		 * When PG_M is already clear, PG_RW can be cleared without
+		 * a TLB invalidation.
+		 */
+		if (!atomic_cmpset_long(firstpte, newpde, newpde & ~PG_RW))
+			goto setpde;
 		newpde &= ~PG_RW;
+	}
 
-	/* 
-	 * Check all the ptes before promotion
+	/*
+	 * Examine each of the other PTEs in the specified PTP.  Abort if this
+	 * PTE maps an unexpected physical page or does not have identical
+	 * characteristics to the first PTE.
 	 */
-	pa = newpde & PG_PS_FRAME;
-	for (pte = firstpte; pte < firstpte + NPTEPG; pte++) {
-retry:
+	pa = (newpde & PG_PS_FRAME) + NBPDR - PAGE_SIZE;
+	for (pte = firstpte + NPTEPG - 1; pte > firstpte; pte--) {
+setpte:
 		oldpte = *pte;
 		if ((oldpte & PG_FRAME) != pa) {
 			pmap_pde_p_failures++;
@@ -2804,7 +2820,7 @@ retry:
 			 * without a TLB invalidation.
 			 */
 			if (!atomic_cmpset_long(pte, oldpte, oldpte & ~PG_RW))
-				goto retry;
+				goto setpte;
 			oldpte &= ~PG_RW;
 			oldpteva = (oldpte & PG_FRAME & PDRMASK) |
 			    (va & ~PDRMASK);
@@ -2817,7 +2833,7 @@ retry:
 			    " in pmap %p", va, pmap);
 			return;
 		}
-		pa += PAGE_SIZE;
+		pa -= PAGE_SIZE;
 	}
 
 	/*