From 44ec2b63c50ae6d0b2765d5cbdbffb0beb276ad4 Mon Sep 17 00:00:00 2001
From: Konstantin Belousov <kib@FreeBSD.org>
Date: Sat, 9 May 2015 20:08:36 +0000
Subject: [PATCH] The vmem callback to reclaim kmem arena address space on low
 or fragmented conditions currently just wakes up the pagedaemon.  The kmem
 arena is significantly smaller then the total available physical memory,
 which means that there are loads where kmem arena space could be exhausted,
 while there is a lot of pages available still.  The woken up pagedaemon sees
 vm_pages_needed != 0, verifies the condition vm_paging_needed() which is
 false, clears the pass and returns back to sleep, not calling neither
 uma_reclaim() nor lowmem handler.

To handle low kmem arena conditions, create additional pagedaemon
thread which calls uma_reclaim() directly.  The thread sleeps on the
dedicated channel and kmem_reclaim() wakes the thread in addition to
the pagedaemon.

Reported and tested by:	pho
Sponsored by:	The FreeBSD Foundation
MFC after:	2 weeks
---
 sys/kern/kern_malloc.c |  6 ++++--
 sys/vm/uma.h           |  3 +++
 sys/vm/uma_core.c      | 42 ++++++++++++++++++++++++++++++++++++++----
 sys/vm/vm_pageout.c    |  7 ++++++-
 4 files changed, 51 insertions(+), 7 deletions(-)

diff --git a/sys/kern/kern_malloc.c b/sys/kern/kern_malloc.c
index aef1e4e23d3..ff5b106e385 100644
--- a/sys/kern/kern_malloc.c
+++ b/sys/kern/kern_malloc.c
@@ -665,13 +665,15 @@ reallocf(void *addr, unsigned long size, struct malloc_type *mtp, int flags)
 }
 
 /*
- * Wake the page daemon when we exhaust KVA.  It will call the lowmem handler
- * and uma_reclaim() callbacks in a context that is safe.
+ * Wake the uma reclamation pagedaemon thread when we exhaust KVA.  It
+ * will call the lowmem handler and uma_reclaim() callbacks in a
+ * context that is safe.
  */
 static void
 kmem_reclaim(vmem_t *vm, int flags)
 {
 
+	uma_reclaim_wakeup();
 	pagedaemon_wakeup();
 }
 
diff --git a/sys/vm/uma.h b/sys/vm/uma.h
index df6cc5c7a11..d3e0658aef1 100644
--- a/sys/vm/uma.h
+++ b/sys/vm/uma.h
@@ -690,4 +690,7 @@ struct uma_percpu_stat {
 	uint64_t	_ups_reserved[5];	/* Reserved. */
 };
 
+void uma_reclaim_wakeup(void);
+void uma_reclaim_worker(void *);
+
 #endif	/* _VM_UMA_H_ */
diff --git a/sys/vm/uma_core.c b/sys/vm/uma_core.c
index 4ff177f8129..d1573e68beb 100644
--- a/sys/vm/uma_core.c
+++ b/sys/vm/uma_core.c
@@ -3222,16 +3222,17 @@ uma_find_refcnt(uma_zone_t zone, void *item)
 }
 
 /* See uma.h */
-void
-uma_reclaim(void)
+static void
+uma_reclaim_locked(bool kmem_danger)
 {
+
 #ifdef UMA_DEBUG
 	printf("UMA: vm asked us to release pages!\n");
 #endif
-	sx_xlock(&uma_drain_lock);
+	sx_assert(&uma_drain_lock, SA_XLOCKED);
 	bucket_enable();
 	zone_foreach(zone_drain);
-	if (vm_page_count_min()) {
+	if (vm_page_count_min() || kmem_danger) {
 		cache_drain_safe(NULL);
 		zone_foreach(zone_drain);
 	}
@@ -3243,9 +3244,42 @@ uma_reclaim(void)
 	zone_drain(slabzone);
 	zone_drain(slabrefzone);
 	bucket_zone_drain();
+}
+
+void
+uma_reclaim(void)
+{
+
+	sx_xlock(&uma_drain_lock);
+	uma_reclaim_locked(false);
 	sx_xunlock(&uma_drain_lock);
 }
 
+static int uma_reclaim_needed;
+
+void
+uma_reclaim_wakeup(void)
+{
+
+	uma_reclaim_needed = 1;
+	wakeup(&uma_reclaim_needed);
+}
+
+void
+uma_reclaim_worker(void *arg __unused)
+{
+
+	sx_xlock(&uma_drain_lock);
+	for (;;) {
+		sx_sleep(&uma_reclaim_needed, &uma_drain_lock, PVM,
+		    "umarcl", 0);
+		if (uma_reclaim_needed) {
+			uma_reclaim_needed = 0;
+			uma_reclaim_locked(true);
+		}
+	}
+}
+
 /* See uma.h */
 int
 uma_zone_exhausted(uma_zone_t zone)
diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c
index 872ffd8f6a0..2f57579da35 100644
--- a/sys/vm/vm_pageout.c
+++ b/sys/vm/vm_pageout.c
@@ -1724,8 +1724,9 @@ vm_pageout_init(void)
 static void
 vm_pageout(void)
 {
+	int error;
 #if MAXMEMDOM > 1
-	int error, i;
+	int i;
 #endif
 
 	swap_pager_swap_init();
@@ -1739,6 +1740,10 @@ vm_pageout(void)
 		}
 	}
 #endif
+	error = kthread_add(uma_reclaim_worker, NULL, curproc, NULL,
+	    0, 0, "uma");
+	if (error != 0)
+		panic("starting uma_reclaim helper, error %d\n", error);
 	vm_pageout_worker((void *)(uintptr_t)0);
 }