From ed01d9894e17a263a833acc03959cd1ccd4b6c2f Mon Sep 17 00:00:00 2001 From: Konstantin Belousov Date: Thu, 8 Dec 2016 11:35:53 +0000 Subject: [PATCH] Implement the populate() pager method for phys pager. It allows to provide configurable agressive prefaulting and useful hints to page daemon about memory allocations, on faults for pages managed by phys pager. In fact, this implementation is superior to the MAP_SHARED_PHYS hack from my Postgresql paper, while giving similar benefits of reducing the page faults numbers on SysV shared memory mappings. Reviewed by: alc Tested by: pho Sponsored by: The FreeBSD Foundation MFC after: 3 weeks --- sys/vm/phys_pager.c | 97 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 85 insertions(+), 12 deletions(-) diff --git a/sys/vm/phys_pager.c b/sys/vm/phys_pager.c index f3b23b09542..7adc7c67585 100644 --- a/sys/vm/phys_pager.c +++ b/sys/vm/phys_pager.c @@ -41,6 +41,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include /* list of phys pager objects */ @@ -98,6 +99,7 @@ phys_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, object = object1; object1 = NULL; object->handle = handle; + vm_object_set_flag(object, OBJ_POPULATE); TAILQ_INSERT_TAIL(&phys_pager_object_list, object, pager_object_list); } @@ -109,6 +111,7 @@ phys_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, vm_object_deallocate(object1); } else { object = vm_object_allocate(OBJT_PHYS, pindex); + vm_object_set_flag(object, OBJ_POPULATE); } return (object); @@ -157,6 +160,85 @@ phys_pager_getpages(vm_object_t object, vm_page_t *m, int count, int *rbehind, return (VM_PAGER_OK); } +/* + * Implement a pretty aggressive clustered getpages strategy. Hint that + * everything in an entire 4MB window should be prefaulted at once. + * + * 4MB (1024 slots per page table page) is convenient for x86, + * but may not be for other arches. + */ +#ifndef PHYSCLUSTER +#define PHYSCLUSTER 1024 +#endif +static int phys_pager_cluster = PHYSCLUSTER; +SYSCTL_INT(_vm, OID_AUTO, phys_pager_cluster, CTLFLAG_RWTUN, + &phys_pager_cluster, 0, + "prefault window size for phys pager"); + +/* + * Max hint to vm_page_alloc() about the further allocation needs + * inside the phys_pager_populate() loop. The number of bits used to + * implement VM_ALLOC_COUNT() determines the hard limit on this value. + * That limit is currently 65535. + */ +#define PHYSALLOC 16 + +static int +phys_pager_populate(vm_object_t object, vm_pindex_t pidx, + int fault_type __unused, vm_prot_t max_prot __unused, vm_pindex_t *first, + vm_pindex_t *last) +{ + vm_page_t m; + vm_pindex_t base, end, i; + int ahead; + + base = rounddown(pidx, phys_pager_cluster); + end = base + phys_pager_cluster - 1; + if (end >= object->size) + end = object->size - 1; + if (*first > base) + base = *first; + if (end > *last) + end = *last; + *first = base; + *last = end; + + for (i = base; i <= end; i++) { +retry: + m = vm_page_lookup(object, i); + if (m == NULL) { + ahead = MIN(end - i, PHYSALLOC); + m = vm_page_alloc(object, i, VM_ALLOC_NORMAL | + VM_ALLOC_ZERO | VM_ALLOC_COUNT(ahead)); + if (m == NULL) { + VM_OBJECT_WUNLOCK(object); + VM_WAIT; + VM_OBJECT_WLOCK(object); + goto retry; + } + if ((m->flags & PG_ZERO) == 0) + pmap_zero_page(m); + m->valid = VM_PAGE_BITS_ALL; + } else if (vm_page_xbusied(m)) { + vm_page_lock(m); + VM_OBJECT_WUNLOCK(object); + vm_page_busy_sleep(m, "physb", true); + VM_OBJECT_WLOCK(object); + goto retry; + } else { + vm_page_xbusy(m); + if (m->valid != VM_PAGE_BITS_ALL) + vm_page_zero_invalid(m, TRUE); + } + + KASSERT(m->valid == VM_PAGE_BITS_ALL, + ("phys_pager_populate: partially valid page %p", m)); + KASSERT(m->dirty == 0, + ("phys_pager_populate: dirty page %p", m)); + } + return (VM_PAGER_OK); +} + static void phys_pager_putpages(vm_object_t object, vm_page_t *m, int count, boolean_t sync, int *rtvals) @@ -165,24 +247,14 @@ phys_pager_putpages(vm_object_t object, vm_page_t *m, int count, boolean_t sync, panic("phys_pager_putpage called"); } -/* - * Implement a pretty aggressive clustered getpages strategy. Hint that - * everything in an entire 4MB window should be prefaulted at once. - * - * XXX 4MB (1024 slots per page table page) is convenient for x86, - * but may not be for other arches. - */ -#ifndef PHYSCLUSTER -#define PHYSCLUSTER 1024 -#endif static boolean_t phys_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after) { vm_pindex_t base, end; - base = rounddown2(pindex, PHYSCLUSTER); - end = base + (PHYSCLUSTER - 1); + base = rounddown(pindex, phys_pager_cluster); + end = base + phys_pager_cluster - 1; if (before != NULL) *before = pindex - base; if (after != NULL) @@ -197,4 +269,5 @@ struct pagerops physpagerops = { .pgo_getpages = phys_pager_getpages, .pgo_putpages = phys_pager_putpages, .pgo_haspage = phys_pager_haspage, + .pgo_populate = phys_pager_populate, };