From 89ea39a72715162b873e030e773285af68eedb2f Mon Sep 17 00:00:00 2001
From: Alan Cox <alc@FreeBSD.org>
Date: Tue, 26 Jun 2018 18:29:56 +0000
Subject: [PATCH] Update the physical page selection strategy used by
 vm_page_import() so that it does not cause rapid fragmentation of the free
 physical memory.

Reviewed by:	jeff, markj (an earlier version)
Differential Revision:	https://reviews.freebsd.org/D15976
---
 sys/vm/vm_page.c | 14 ++------
 sys/vm/vm_phys.c | 90 +++++++++++++++++++++++++++++++++++++-----------
 sys/vm/vm_phys.h |  2 +-
 3 files changed, 74 insertions(+), 32 deletions(-)

diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index bf65f292339..155512449df 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -2235,24 +2235,16 @@ static int
 vm_page_import(void *arg, void **store, int cnt, int domain, int flags)
 {
 	struct vm_domain *vmd;
-	vm_page_t m;
-	int i, j, n;
+	int i;
 
 	vmd = arg;
 	/* Only import if we can bring in a full bucket. */
 	if (cnt == 1 || !vm_domain_allocate(vmd, VM_ALLOC_NORMAL, cnt))
 		return (0);
 	domain = vmd->vmd_domain;
-	n = 64;	/* Starting stride, arbitrary. */
 	vm_domain_free_lock(vmd);
-	for (i = 0; i < cnt; i+=n) {
-		n = vm_phys_alloc_npages(domain, VM_FREELIST_DEFAULT, &m,
-		    MIN(n, cnt-i));
-		if (n == 0)
-			break;
-		for (j = 0; j < n; j++)
-			store[i+j] = m++;
-	}
+	i = vm_phys_alloc_npages(domain, VM_FREEPOOL_DEFAULT, cnt,
+	    (vm_page_t *)store);
 	vm_domain_free_unlock(vmd);
 	if (cnt != i)
 		vm_domain_freecnt_inc(vmd, cnt - i);
diff --git a/sys/vm/vm_phys.c b/sys/vm/vm_phys.c
index 0aaa9e7a960..4de5c2bf290 100644
--- a/sys/vm/vm_phys.c
+++ b/sys/vm/vm_phys.c
@@ -604,6 +604,76 @@ vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl, int order)
         }
 }
 
+/*
+ * Tries to allocate the specified number of pages from the specified pool
+ * within the specified domain.  Returns the actual number of allocated pages
+ * and a pointer to each page through the array ma[].
+ *
+ * The returned pages may not be physically contiguous.  However, in contrast to
+ * performing multiple, back-to-back calls to vm_phys_alloc_pages(..., 0),
+ * calling this function once to allocate the desired number of pages will avoid
+ * wasted time in vm_phys_split_pages().
+ *
+ * The free page queues for the specified domain must be locked.
+ */
+int
+vm_phys_alloc_npages(int domain, int pool, int npages, vm_page_t ma[])
+{
+	struct vm_freelist *alt, *fl;
+	vm_page_t m;
+	int avail, end, flind, freelist, i, need, oind, pind;
+
+	KASSERT(domain >= 0 && domain < vm_ndomains,
+	    ("vm_phys_alloc_npages: domain %d is out of range", domain));
+	KASSERT(pool < VM_NFREEPOOL,
+	    ("vm_phys_alloc_npages: pool %d is out of range", pool));
+	KASSERT(npages <= 1 << (VM_NFREEORDER - 1),
+	    ("vm_phys_alloc_npages: npages %d is out of range", npages));
+	vm_domain_free_assert_locked(VM_DOMAIN(domain));
+	i = 0;
+	for (freelist = 0; freelist < VM_NFREELIST; freelist++) {
+		flind = vm_freelist_to_flind[freelist];
+		if (flind < 0)
+			continue;
+		fl = vm_phys_free_queues[domain][flind][pool];
+		for (oind = 0; oind < VM_NFREEORDER; oind++) {
+			while ((m = TAILQ_FIRST(&fl[oind].pl)) != NULL) {
+				vm_freelist_rem(fl, m, oind);
+				avail = 1 << oind;
+				need = imin(npages - i, avail);
+				for (end = i + need; i < end;)
+					ma[i++] = m++;
+				if (need < avail) {
+					vm_phys_free_contig(m, avail - need);
+					return (npages);
+				} else if (i == npages)
+					return (npages);
+			}
+		}
+		for (oind = VM_NFREEORDER - 1; oind >= 0; oind--) {
+			for (pind = 0; pind < VM_NFREEPOOL; pind++) {
+				alt = vm_phys_free_queues[domain][flind][pind];
+				while ((m = TAILQ_FIRST(&alt[oind].pl)) !=
+				    NULL) {
+					vm_freelist_rem(alt, m, oind);
+					vm_phys_set_pool(pool, m, oind);
+					avail = 1 << oind;
+					need = imin(npages - i, avail);
+					for (end = i + need; i < end;)
+						ma[i++] = m++;
+					if (need < avail) {
+						vm_phys_free_contig(m, avail -
+						    need);
+						return (npages);
+					} else if (i == npages)
+						return (npages);
+				}
+			}
+		}
+	}
+	return (i);
+}
+
 /*
  * Allocate a contiguous, power of two-sized set of physical pages
  * from the free lists.
@@ -624,26 +694,6 @@ vm_phys_alloc_pages(int domain, int pool, int order)
 	return (NULL);
 }
 
-int
-vm_phys_alloc_npages(int domain, int pool, vm_page_t *mp, int cnt)
-{
-	vm_page_t m;
-	int order, freelist;
-
-	for (freelist = 0; freelist < VM_NFREELIST; freelist++) {
-		for (order = fls(cnt) -1; order >= 0; order--) {
-			m = vm_phys_alloc_freelist_pages(domain, freelist,
-			    pool, order);
-			if (m != NULL) {
-				*mp = m;
-				return (1 << order);
-			}
-		}
-	}
-	*mp = NULL;
-	return (0);
-}
-
 /*
  * Allocate a contiguous, power of two-sized set of physical pages from the
  * specified free list.  The free list must be specified using one of the
diff --git a/sys/vm/vm_phys.h b/sys/vm/vm_phys.h
index c7ec5119981..a01a4a82595 100644
--- a/sys/vm/vm_phys.h
+++ b/sys/vm/vm_phys.h
@@ -77,8 +77,8 @@ vm_page_t vm_phys_alloc_contig(int domain, u_long npages, vm_paddr_t low,
     vm_paddr_t high, u_long alignment, vm_paddr_t boundary);
 vm_page_t vm_phys_alloc_freelist_pages(int domain, int freelist, int pool,
     int order);
+int vm_phys_alloc_npages(int domain, int pool, int npages, vm_page_t ma[]);
 vm_page_t vm_phys_alloc_pages(int domain, int pool, int order);
-int vm_phys_alloc_npages(int domain, int pool, vm_page_t *m, int cnt);
 int vm_phys_domain_match(int prefer, vm_paddr_t low, vm_paddr_t high);
 int vm_phys_fictitious_reg_range(vm_paddr_t start, vm_paddr_t end,
     vm_memattr_t memattr);