From 16147166550303ef4e55db8ebea57df6e7caeba7 Mon Sep 17 00:00:00 2001
From: Konstantin Belousov <kib@FreeBSD.org>
Date: Fri, 6 Jul 2018 19:48:47 +0000
Subject: [PATCH] Save a call to pmap_remove() if entry cannot have any pages
 mapped.

Due to the way rtld creates mappings for the shared objects, each dso
causes unmap of at least three guard map entries.  For instance, in
the buildworld load, this change reduces the amount of pmap_remove()
calls by 1/5.

Profiled by:	alc
Reviewed by:	alc, markj
Sponsored by:	The FreeBSD Foundation
MFC after:	1 week
Differential revision:	https://reviews.freebsd.org/D16148
---
 sys/amd64/include/counter.h |  9 +--------
 sys/amd64/include/pcpu.h    |  3 +--
 sys/i386/include/pcpu.h     |  2 +-
 sys/sys/pcpu.h              | 21 +++++++++++++--------
 4 files changed, 16 insertions(+), 19 deletions(-)

diff --git a/sys/amd64/include/counter.h b/sys/amd64/include/counter.h
index b09790fcb34..2255a0575a0 100644
--- a/sys/amd64/include/counter.h
+++ b/sys/amd64/include/counter.h
@@ -83,18 +83,11 @@ counter_u64_zero_inline(counter_u64_t c)
 static inline void
 counter_u64_add(counter_u64_t c, int64_t inc)
 {
-	int64_t *p;
-#ifdef notyet
+
 	__asm __volatile("addq\t%1,%%gs:(%0)"
 	    :
 	    : "r" ((char *)c - (char *)&__pcpu[0]), "ri" (inc)
 	    : "memory", "cc");
-#endif
-	/* temporary */
-	critical_enter();
-	p = zpcpu_get(c);
-	*p += inc;
-	critical_exit();
 }
 
 #endif	/* ! __MACHINE_COUNTER_H__ */
diff --git a/sys/amd64/include/pcpu.h b/sys/amd64/include/pcpu.h
index 0cd476a6cae..f1493176708 100644
--- a/sys/amd64/include/pcpu.h
+++ b/sys/amd64/include/pcpu.h
@@ -76,8 +76,7 @@
 	uint32_t pc_pcid_gen;						\
 	uint32_t pc_smp_tlb_done;	/* TLB op acknowledgement */	\
 	uint32_t pc_ibpb_set;						\
-	char	__pad[216]		/* be divisor of PAGE_SIZE	\
-					   after cache alignment */
+	char	__pad[3288]		/* pad to UMA_PCPU_ALLOC_SIZE */
 
 #define	PC_DBREG_CMD_NONE	0
 #define	PC_DBREG_CMD_LOAD	1
diff --git a/sys/i386/include/pcpu.h b/sys/i386/include/pcpu.h
index be4594963fe..50864d7d364 100644
--- a/sys/i386/include/pcpu.h
+++ b/sys/i386/include/pcpu.h
@@ -80,7 +80,7 @@
 	caddr_t pc_pmap_eh_ptep;						\
 	uint32_t pc_smp_tlb_done;	/* TLB op acknowledgement */	\
 	uint32_t pc_ibpb_set;						\
-	char	__pad[538]
+	char	__pad[3610]
 
 #ifdef _KERNEL
 
diff --git a/sys/sys/pcpu.h b/sys/sys/pcpu.h
index 9ba2adfdbff..cba9d6e8979 100644
--- a/sys/sys/pcpu.h
+++ b/sys/sys/pcpu.h
@@ -185,14 +185,6 @@ struct pcpu {
 	PCPU_MD_FIELDS;
 } __aligned(CACHE_LINE_SIZE);
 
-#ifdef CTASSERT
-/*
- * To minimize memory waste in per-cpu UMA zones, size of struct pcpu
- * should be denominator of PAGE_SIZE.
- */
-CTASSERT((PAGE_SIZE / sizeof(struct pcpu)) * sizeof(struct pcpu) == PAGE_SIZE);
-#endif
-
 #ifdef _KERNEL
 
 STAILQ_HEAD(cpuhead, pcpu);
@@ -209,6 +201,19 @@ extern struct pcpu *cpuid_to_pcpu[];
 
 #define UMA_PCPU_ALLOC_SIZE		PAGE_SIZE
 
+#ifdef CTASSERT
+#if defined(__i386__) || defined(__amd64__)
+/* Required for counters(9) to work on x86. */
+CTASSERT(sizeof(struct pcpu) == UMA_PCPU_ALLOC_SIZE);
+#else
+/*
+ * To minimize memory waste in per-cpu UMA zones, size of struct pcpu
+ * should be denominator of PAGE_SIZE.
+ */
+CTASSERT((PAGE_SIZE / sizeof(struct pcpu)) * sizeof(struct pcpu) == PAGE_SIZE);
+#endif	/* UMA_PCPU_ALLOC_SIZE && x86 */
+#endif	/* CTASSERT */
+
 /* Accessor to elements allocated via UMA_ZONE_PCPU zone. */
 static inline void *
 zpcpu_get(void *base)