upstream/opnsense-src
1
0
Fork 0
mirror of https://github.com/opnsense/src.git synced 2026-06-13 18:50:31 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

vm_page: make iter_insert() public

Browse source 
In places where vm_page_insert() is used after lookups, or for
consecutive pages, use vm_page_iter_insert instead, to exploit
locality.

Reviewed by:	kib, markj
Differential Revision:	https://reviews.freebsd.org/D50248
This commit is contained in:
Doug Moore 2025-05-08 23:13:32 -05:00
parent 8703b572b9
commit bcd85e0150
6 changed files with 50 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

62
sys/amd64/sgx/sgx.c
View file

@ -191,9 +191,20 @@ sgx_put_epc_page(struct sgx_softc *sc, struct epc_page *epc)
vmem_free(sc->vmem_epc, addr, PAGE_SIZE);
}
static void
sgx_insert_epc_page_by_index(vm_page_t page, vm_object_t object,
vm_pindex_t pidx, struct pctrie_iter *pages)
{
VM_OBJECT_ASSERT_WLOCKED(object);
page->valid = VM_PAGE_BITS_ALL;
vm_page_iter_insert(page, object, pidx, pages);
}
static int
sgx_va_slot_init_by_index(struct sgx_softc *sc, vm_object_t object,
uint64_t idx)
uint64_t idx, struct pctrie_iter *pages)
{
struct epc_page *epc;
vm_page_t page;
@ -202,7 +213,7 @@ sgx_va_slot_init_by_index(struct sgx_softc *sc, vm_object_t object,
VM_OBJECT_ASSERT_WLOCKED(object);
p = vm_page_lookup(object, idx);
p = vm_radix_iter_lookup(pages, idx);
if (p == NULL) {
ret = sgx_get_epc_page(sc, &epc);
if (ret) {
@ -216,20 +227,16 @@ sgx_va_slot_init_by_index(struct sgx_softc *sc, vm_object_t object,
mtx_unlock(&sc->mtx_encls);
page = PHYS_TO_VM_PAGE(epc->phys);
page->valid = VM_PAGE_BITS_ALL;
vm_page_insert(page, object, idx);
sgx_insert_epc_page_by_index(page, object, idx, pages);
}
return (0);
}
static int
sgx_va_slot_init(struct sgx_softc *sc,
struct sgx_enclave *enclave,
uint64_t addr)
sgx_va_slot_init(struct sgx_softc *sc, struct sgx_enclave *enclave,
vm_pindex_t pidx, struct pctrie_iter *pages)
{
vm_pindex_t pidx;
uint64_t va_page_idx;
uint64_t idx;
vm_object_t object;
@ -239,12 +246,10 @@ sgx_va_slot_init(struct sgx_softc *sc,
VM_OBJECT_ASSERT_WLOCKED(object);
pidx = OFF_TO_IDX(addr);
va_page_idx = pidx / SGX_VA_PAGE_SLOTS;
idx = - SGX_VA_PAGES_OFFS - va_page_idx;
ret = sgx_va_slot_init_by_index(sc, object, idx);
ret = sgx_va_slot_init_by_index(sc, object, idx, pages);
return (ret);
}
@ -614,19 +619,8 @@ static struct cdev_pager_ops sgx_pg_ops = {
};
static void
sgx_insert_epc_page_by_index(vm_page_t page, vm_object_t object,
vm_pindex_t pidx)
{
VM_OBJECT_ASSERT_WLOCKED(object);
page->valid = VM_PAGE_BITS_ALL;
vm_page_insert(page, object, pidx);
}
static void
sgx_insert_epc_page(struct sgx_enclave *enclave,
struct epc_page *epc, uint64_t addr)
sgx_insert_epc_page(struct sgx_enclave *enclave, struct epc_page *epc,
uint64_t addr, struct pctrie_iter *pages)
{
vm_pindex_t pidx;
vm_page_t page;
@ -636,12 +630,13 @@ sgx_insert_epc_page(struct sgx_enclave *enclave,
pidx = OFF_TO_IDX(addr);
page = PHYS_TO_VM_PAGE(epc->phys);
sgx_insert_epc_page_by_index(page, enclave->object, pidx);
sgx_insert_epc_page_by_index(page, enclave->object, pidx, pages);
}
static int
sgx_ioctl_create(struct sgx_softc *sc, struct sgx_enclave_create *param)
{
struct pctrie_iter pages;
struct sgx_vm_handle *vmh;
vm_map_entry_t entry;
vm_page_t p;
@ -712,8 +707,9 @@ sgx_ioctl_create(struct sgx_softc *sc, struct sgx_enclave_create *param)
}
enclave->secs_epc_page = epc;
vm_page_iter_init(&pages, object);
VM_OBJECT_WLOCK(object);
p = vm_page_lookup(object, SGX_SECS_VM_OBJECT_INDEX);
p = vm_radix_iter_lookup(&pages, SGX_SECS_VM_OBJECT_INDEX);
if (p) {
VM_OBJECT_WUNLOCK(object);
/* SECS page already added. */
@ -722,7 +718,7 @@ sgx_ioctl_create(struct sgx_softc *sc, struct sgx_enclave_create *param)
}
ret = sgx_va_slot_init_by_index(sc, object,
- SGX_VA_PAGES_OFFS - SGX_SECS_VM_OBJECT_INDEX);
- SGX_VA_PAGES_OFFS - SGX_SECS_VM_OBJECT_INDEX, &pages);
if (ret) {
VM_OBJECT_WUNLOCK(object);
dprintf("%s: Can't init va slot.\n", __func__);
@ -762,7 +758,7 @@ sgx_ioctl_create(struct sgx_softc *sc, struct sgx_enclave_create *param)
page = PHYS_TO_VM_PAGE(epc->phys);
sgx_insert_epc_page_by_index(page, enclave->object,
SGX_SECS_VM_OBJECT_INDEX);
SGX_SECS_VM_OBJECT_INDEX, &pages);
VM_OBJECT_WUNLOCK(object);
@ -786,6 +782,7 @@ static int
sgx_ioctl_add_page(struct sgx_softc *sc,
struct sgx_enclave_add_page *addp)
{
struct pctrie_iter pages;
struct epc_page *secs_epc_page;
struct sgx_enclave *enclave;
struct sgx_vm_handle *vmh;
@ -857,8 +854,9 @@ sgx_ioctl_add_page(struct sgx_softc *sc,
addr = (addp->addr - vmh->base);
pidx = OFF_TO_IDX(addr);
vm_page_iter_init(&pages, object);
VM_OBJECT_WLOCK(object);
p = vm_page_lookup(object, pidx);
p = vm_radix_iter_lookup(&pages, pidx);
if (p) {
VM_OBJECT_WUNLOCK(object);
/* Page already added. */
@ -866,7 +864,7 @@ sgx_ioctl_add_page(struct sgx_softc *sc,
goto error;
}
ret = sgx_va_slot_init(sc, enclave, addr);
ret = sgx_va_slot_init(sc, enclave, pidx, &pages);
if (ret) {
VM_OBJECT_WUNLOCK(object);
dprintf("%s: Can't init va slot.\n", __func__);
@ -898,7 +896,7 @@ sgx_ioctl_add_page(struct sgx_softc *sc,
goto error;
}
sgx_insert_epc_page(enclave, epc, addr);
sgx_insert_epc_page(enclave, epc, addr, &pages);
VM_OBJECT_WUNLOCK(object);

4
sys/arm/nvidia/drm2/tegra_bo.c
View file

@ -133,6 +133,7 @@ retry:
static int
tegra_bo_init_pager(struct tegra_bo *bo)
{
struct pctrie_iter pages;
vm_page_t m;
size_t size;
int i;
@ -143,6 +144,7 @@ tegra_bo_init_pager(struct tegra_bo *bo)
if (vmem_alloc(kernel_arena, size, M_WAITOK | M_BESTFIT, &bo->vbase))
return (ENOMEM);
vm_page_iter_init(&pages, bo->cdev_pager);
VM_OBJECT_WLOCK(bo->cdev_pager);
for (i = 0; i < bo->npages; i++) {
m = bo->m[i];
@ -159,7 +161,7 @@ tegra_bo_init_pager(struct tegra_bo *bo)
*/
m->oflags &= ~VPO_UNMANAGED;
m->flags |= PG_FICTITIOUS;
if (vm_page_insert(m, bo->cdev_pager, i) != 0)
if (vm_page_iter_insert(m, bo->cdev_pager, i, &pages) != 0)
return (EINVAL);
}
VM_OBJECT_WUNLOCK(bo->cdev_pager);

11
sys/compat/linuxkpi/common/src/linux_page.c
View file

@ -296,23 +296,27 @@ vm_fault_t
lkpi_vmf_insert_pfn_prot_locked(struct vm_area_struct *vma, unsigned long addr,
unsigned long pfn, pgprot_t prot)
{
struct pctrie_iter pages;
vm_object_t vm_obj = vma->vm_obj;
vm_object_t tmp_obj;
vm_page_t page;
vm_pindex_t pindex;
VM_OBJECT_ASSERT_WLOCKED(vm_obj);
vm_page_iter_init(&pages, vm_obj);
pindex = OFF_TO_IDX(addr - vma->vm_start);
if (vma->vm_pfn_count == 0)
vma->vm_pfn_first = pindex;
MPASS(pindex <= OFF_TO_IDX(vma->vm_end));
retry:
page = vm_page_grab(vm_obj, pindex, VM_ALLOC_NOCREAT);
page = vm_page_grab_iter(vm_obj, pindex, VM_ALLOC_NOCREAT, &pages);
if (page == NULL) {
page = PHYS_TO_VM_PAGE(IDX_TO_OFF(pfn));
if (!vm_page_busy_acquire(page, VM_ALLOC_WAITFAIL))
if (!vm_page_busy_acquire(page, VM_ALLOC_WAITFAIL)) {
pctrie_iter_reset(&pages);
goto retry;
}
if (page->object != NULL) {
tmp_obj = page->object;
vm_page_xunbusy(page);
@ -336,10 +340,11 @@ retry:
vm_page_remove(page);
}
VM_OBJECT_WUNLOCK(tmp_obj);
pctrie_iter_reset(&pages);
VM_OBJECT_WLOCK(vm_obj);
goto retry;
}
if (vm_page_insert(page, vm_obj, pindex)) {
if (vm_page_iter_insert(page, vm_obj, pindex, &pages) != 0) {
vm_page_xunbusy(page);
return (VM_FAULT_OOM);
}

9
sys/dev/drm2/ttm/ttm_bo_vm.c
View file

@ -100,7 +100,7 @@ static int
ttm_bo_vm_fault(vm_object_t vm_obj, vm_ooffset_t offset,
int prot, vm_page_t *mres)
{
struct pctrie_iter pages;
struct ttm_buffer_object *bo = vm_obj->handle;
struct ttm_bo_device *bdev = bo->bdev;
struct ttm_tt *ttm = NULL;
@ -114,6 +114,7 @@ ttm_bo_vm_fault(vm_object_t vm_obj, vm_ooffset_t offset,
if (*mres != NULL) {
(void)vm_page_remove(*mres);
}
vm_page_iter_init(&pages, vm_obj);
retry:
VM_OBJECT_WUNLOCK(vm_obj);
m = NULL;
@ -234,10 +235,12 @@ reserve:
ttm_bo_unreserve(bo);
goto retry;
}
m1 = vm_page_lookup(vm_obj, OFF_TO_IDX(offset));
pctrie_iter_reset(&pages);
m1 = vm_radix_iter_lookup(&pages, OFF_TO_IDX(offset));
/* XXX This looks like it should just be vm_page_replace? */
if (m1 == NULL) {
if (vm_page_insert(m, vm_obj, OFF_TO_IDX(offset))) {
if (vm_page_iter_insert(
m, vm_obj, OFF_TO_IDX(offset), &pages) != 0) {
vm_page_xunbusy(m);
VM_OBJECT_WUNLOCK(vm_obj);
vm_wait(vm_obj);

2
sys/vm/vm_page.c
View file

@ -1532,7 +1532,7 @@ vm_page_insert(vm_page_t m, vm_object_t object, vm_pindex_t pindex)
*
* The object must be locked.
*/
static int
int
vm_page_iter_insert(vm_page_t m, vm_object_t object, vm_pindex_t pindex,
struct pctrie_iter *pages)
{

2
sys/vm/vm_page.h
View file

@ -653,6 +653,8 @@ int vm_page_insert (vm_page_t, vm_object_t, vm_pindex_t);
void vm_page_invalid(vm_page_t m);
void vm_page_iter_free(struct pctrie_iter *pages, vm_page_t m);
void vm_page_iter_init(struct pctrie_iter *, vm_object_t);
int vm_page_iter_insert(vm_page_t m, vm_object_t, vm_pindex_t,
struct pctrie_iter *);
void vm_page_iter_limit_init(struct pctrie_iter *, vm_object_t, vm_pindex_t);
bool vm_page_iter_remove(struct pctrie_iter *pages, vm_page_t m);
bool vm_page_iter_rename(struct pctrie_iter *old_pages, vm_page_t m,