upstream/opnsense-src
1
0
Fork 0
mirror of https://github.com/opnsense/src.git synced 2026-06-07 07:42:26 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

swap_pager: cleanup swapoff_object

Browse source 
Function swap_pager_swapoff_object calls vm_pager_unswapped (via
swp_pager_force_dirty) for every page that must be unswapped. That
means that there's an unneeded check for lock ownership (the caller
always owns it), a needless PCTRIE_LOOKUP (the caller has already
found it), a call to free one page of swap space only, and a check to
see if all blocks are empty, when the caller usually knows that the
check is useless.

Isolate the essential part, needed however swap_pager_unswapped is
invoked, into a smaller function swap_pager_unswapped_acct.  From
swapoff_object, invoke swp_pager_update_freerange for each appropriate
page, so that there are potentially fewer calls to
swp_pager_freeswapspace.  Consider freeing a set of blocks (a struct
swblk) only after having invalidated all those blocks.

Replace the doubly-nested loops with a single loop, and refetch and
rescan a swblk only when the object write lock has been released and
reacquired.

After getting a page from swap, dirty it immediately to address a race
condition observed by @kib.

Reviewed by:	kib
Differential Revision:	https://reviews.freebsd.org/D45668
This commit is contained in:
Doug Moore 2024-06-27 14:06:09 -05:00
parent 14fee5324a
commit 995730a635
1 changed files with 106 additions and 84 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

190
sys/vm/swap_pager.c
View file

@ -1183,6 +1183,21 @@ swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before,
return (TRUE);
}
static void
swap_pager_unswapped_acct(vm_page_t m)
{
KASSERT((m->object->flags & OBJ_SWAP) != 0,
("Free object not swappable"));
if ((m->a.flags & PGA_SWAP_FREE) != 0)
counter_u64_add(swap_free_completed, 1);
vm_page_aflag_clear(m, PGA_SWAP_FREE | PGA_SWAP_SPACE);
/*
* The meta data only exists if the object is OBJT_SWAP
* and even then might not be allocated yet.
*/
}
/*
* SWAP_PAGER_PAGE_UNSWAPPED() - remove swap backing store related to page
*
@ -1229,16 +1244,7 @@ swap_pager_unswapped(vm_page_t m)
}
return;
}
if ((m->a.flags & PGA_SWAP_FREE) != 0)
counter_u64_add(swap_free_completed, 1);
vm_page_aflag_clear(m, PGA_SWAP_FREE | PGA_SWAP_SPACE);
/*
* The meta data only exists if the object is OBJT_SWAP
* and even then might not be allocated yet.
*/
KASSERT((m->object->flags & OBJ_SWAP) != 0,
("Free object not swappable"));
swap_pager_unswapped_acct(m);
sb = SWAP_PCTRIE_LOOKUP(&m->object->un_pager.swp.swp_blks,
rounddown(m->pindex, SWAP_META_PAGES));
@ -1786,11 +1792,12 @@ swap_pager_nswapdev(void)
}
static void
swp_pager_force_dirty(vm_page_t m)
swp_pager_force_dirty(struct page_range *range, vm_page_t m, daddr_t *blk)
{
vm_page_dirty(m);
swap_pager_unswapped(m);
swap_pager_unswapped_acct(m);
swp_pager_update_freerange(range, *blk);
*blk = SWAPBLK_NONE;
vm_page_launder(m);
}
@ -1827,18 +1834,23 @@ swap_pager_swapped_pages(vm_object_t object)
static void
swap_pager_swapoff_object(struct swdevt *sp, vm_object_t object)
{
struct page_range range;
struct swblk *sb;
vm_page_t m;
vm_pindex_t pi;
daddr_t blk;
int i, nv, rahead, rv;
int i, rahead, rv;
bool sb_empty;
VM_OBJECT_ASSERT_WLOCKED(object);
KASSERT((object->flags & OBJ_SWAP) != 0,
("%s: Object not swappable", __func__));
for (pi = 0; (sb = SWAP_PCTRIE_LOOKUP_GE(
&object->un_pager.swp.swp_blks, pi)) != NULL; ) {
if ((object->flags & OBJ_DEAD) != 0) {
pi = 0;
i = 0;
swp_pager_init_freerange(&range);
for (;;) {
if (i == 0 && (object->flags & OBJ_DEAD) != 0) {
/*
* Make sure that pending writes finish before
* returning.
@ -1847,76 +1859,86 @@ swap_pager_swapoff_object(struct swdevt *sp, vm_object_t object)
swp_pager_meta_free_all(object);
break;
}
for (i = 0; i < SWAP_META_PAGES; i++) {
/*
* Count the number of contiguous valid blocks.
*/
for (nv = 0; nv < SWAP_META_PAGES - i; nv++) {
blk = sb->d[i + nv];
if (!swp_pager_isondev(blk, sp) ||
blk == SWAPBLK_NONE)
break;
}
if (nv == 0)
continue;
/*
* Look for a page corresponding to the first
* valid block and ensure that any pending paging
* operations on it are complete. If the page is valid,
* mark it dirty and free the swap block. Try to batch
* this operation since it may cause sp to be freed,
* meaning that we must restart the scan. Avoid busying
* valid pages since we may block forever on kernel
* stack pages.
*/
m = vm_page_lookup(object, sb->p + i);
if (m == NULL) {
m = vm_page_alloc(object, sb->p + i,
VM_ALLOC_NORMAL | VM_ALLOC_WAITFAIL);
if (m == NULL)
break;
} else {
if ((m->oflags & VPO_SWAPINPROG) != 0) {
m->oflags |= VPO_SWAPSLEEP;
VM_OBJECT_SLEEP(object, &object->handle,
PSWP, "swpoff", 0);
break;
}
if (vm_page_all_valid(m)) {
do {
swp_pager_force_dirty(m);
} while (--nv > 0 &&
(m = vm_page_next(m)) != NULL &&
vm_page_all_valid(m) &&
(m->oflags & VPO_SWAPINPROG) == 0);
break;
}
if (!vm_page_busy_acquire(m, VM_ALLOC_WAITFAIL))
break;
}
vm_object_pip_add(object, 1);
rahead = SWAP_META_PAGES;
rv = swap_pager_getpages_locked(object, &m, 1, NULL,
&rahead);
if (rv != VM_PAGER_OK)
panic("%s: read from swap failed: %d",
__func__, rv);
VM_OBJECT_WLOCK(object);
vm_object_pip_wakeupn(object, 1);
vm_page_xunbusy(m);
/*
* The object lock was dropped so we must restart the
* scan of this swap block. Pages paged in during this
* iteration will be marked dirty in a future iteration.
*/
break;
}
if (i == SWAP_META_PAGES)
if (i == SWAP_META_PAGES) {
pi = sb->p + SWAP_META_PAGES;
if (sb_empty) {
SWAP_PCTRIE_REMOVE(
&object->un_pager.swp.swp_blks, sb->p);
uma_zfree(swblk_zone, sb);
}
i = 0;
}
if (i == 0) {
sb = SWAP_PCTRIE_LOOKUP_GE(
&object->un_pager.swp.swp_blks, pi);
if (sb == NULL)
break;
sb_empty = true;
m = NULL;
}
/* Skip an invalid block. */
blk = sb->d[i];
if (blk == SWAPBLK_NONE || !swp_pager_isondev(blk, sp)) {
if (blk != SWAPBLK_NONE)
sb_empty = false;
m = NULL;
i++;
continue;
}
/*
* Look for a page corresponding to this block, If the found
* page has pending operations, sleep and restart the scan.
*/
m = m != NULL ? vm_page_next(m) :
vm_page_lookup(object, sb->p + i);
if (m != NULL && (m->oflags & VPO_SWAPINPROG) != 0) {
m->oflags |= VPO_SWAPSLEEP;
VM_OBJECT_SLEEP(object, &object->handle, PSWP, "swpoff",
0);
i = 0; /* Restart scan after object lock dropped. */
continue;
}
/*
* If the found page is valid, mark it dirty and free the swap
* block.
*/
if (m != NULL && vm_page_all_valid(m)) {
swp_pager_force_dirty(&range, m, &sb->d[i]);
i++;
continue;
}
/* Is there a page we can acquire or allocate? */
if (m == NULL) {
m = vm_page_alloc(object, sb->p + i,
VM_ALLOC_NORMAL | VM_ALLOC_WAITFAIL);
} else if (!vm_page_busy_acquire(m, VM_ALLOC_WAITFAIL))
m = NULL;
/* If no page available, repeat this iteration. */
if (m == NULL)
continue;
/* Get the page from swap, mark it dirty, restart the scan. */
vm_object_pip_add(object, 1);
rahead = SWAP_META_PAGES;
rv = swap_pager_getpages_locked(object, &m, 1, NULL, &rahead);
if (rv != VM_PAGER_OK)
panic("%s: read from swap failed: %d", __func__, rv);
VM_OBJECT_WLOCK(object);
vm_object_pip_wakeupn(object, 1);
KASSERT(vm_page_all_valid(m),
("%s: Page %p not all valid", __func__, m));
swp_pager_force_dirty(&range, m, &sb->d[i]);
vm_page_xunbusy(m);
i = 0; /* Restart scan after object lock dropped. */
}
swp_pager_freeswapspace(&range);
}
/*