From b82149116a2f14f45d3460553f8d514c99d985cc Mon Sep 17 00:00:00 2001 From: Konstantin Belousov Date: Tue, 22 Sep 2020 23:28:06 +0000 Subject: [PATCH] amd64 pmap: More unification for psind = 1 vs 2 in pmap_enter_largepage(). Move pkru check wait for page alloc wire accounting update asserting allowed updates for valid mappings out of psind conditions. Also add assert that psind references supported page size. Remove not true comment. Avoid uneccessary page table walks from top level. Reviewed by: alc, markj (previous version) Sponsored by: The FreeBSD Foundation MFC after: 1 week Differential revision: https://reviews.freebsd.org/D26513 --- sys/amd64/amd64/pmap.c | 77 +++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 49 deletions(-) diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c index 80c665da24d..68cdfba155c 100644 --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -6479,7 +6479,7 @@ pmap_enter_largepage(pmap_t pmap, vm_offset_t va, pt_entry_t newpte, int flags, pt_entry_t origpte, *pml4e, *pdpe, *pde, pten, PG_V; PMAP_LOCK_ASSERT(pmap, MA_OWNED); - KASSERT(psind > 0 && psind < MAXPAGESIZES, + KASSERT(psind > 0 && psind < MAXPAGESIZES && pagesizes[psind] != 0, ("psind %d unexpected", psind)); KASSERT(((newpte & PG_FRAME) & (pagesizes[psind] - 1)) == 0, ("unaligned phys address %#lx newpte %#lx psind %d", @@ -6494,38 +6494,25 @@ pmap_enter_largepage(pmap_t pmap, vm_offset_t va, pt_entry_t newpte, int flags, PG_V = pmap_valid_bit(pmap); restart: + if (!pmap_pkru_same(pmap, va, va + pagesizes[psind])) + return (KERN_PROTECTION_FAILURE); pten = newpte; if (va < VM_MAXUSER_ADDRESS && pmap->pm_type == PT_X86) pten |= pmap_pkru_get(pmap, va); if (psind == 2) { /* 1G */ - if (!pmap_pkru_same(pmap, va, va + NBPDP)) - return (KERN_PROTECTION_FAILURE); pml4e = pmap_pml4e(pmap, va); if (pml4e == NULL || (*pml4e & PG_V) == 0) { mp = _pmap_allocpte(pmap, pmap_pml4e_pindex(va), NULL, va); - if (mp == NULL) { - if ((flags & PMAP_ENTER_NOSLEEP) != 0) - return (KERN_RESOURCE_SHORTAGE); - PMAP_UNLOCK(pmap); - vm_wait(NULL); - PMAP_LOCK(pmap); - - /* - * Restart at least to recalcuate the pkru - * key. Our caller must keep the map locked - * so no paging structure can be validated - * under us. - */ - goto restart; - } - pdpe = pmap_pdpe(pmap, va); - KASSERT(pdpe != NULL, ("va %#lx lost pdpe", va)); + if (mp == NULL) + goto allocf; + pdpe = (pdp_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mp)); + pdpe = &pdpe[pmap_pdpe_index(va)]; origpte = *pdpe; MPASS(origpte == 0); } else { - pdpe = pmap_pdpe(pmap, va); + pdpe = pmap_pml4e_to_pdpe(pml4e, va); KASSERT(pdpe != NULL, ("va %#lx lost pdpe", va)); origpte = *pdpe; if ((origpte & PG_V) == 0) { @@ -6533,57 +6520,49 @@ restart: mp->ref_count++; } } - KASSERT((origpte & PG_V) == 0 || ((origpte & PG_PS) != 0 && - (origpte & PG_FRAME) == (pten & PG_FRAME)), - ("va %#lx changing 1G phys page pdpe %#lx pten %#lx", - va, origpte, pten)); - if ((pten & PG_W) != 0 && (origpte & PG_W) == 0) - pmap->pm_stats.wired_count += NBPDP / PAGE_SIZE; - else if ((pten & PG_W) == 0 && (origpte & PG_W) != 0) - pmap->pm_stats.wired_count -= NBPDP / PAGE_SIZE; *pdpe = pten; } else /* (psind == 1) */ { /* 2M */ - if (!pmap_pkru_same(pmap, va, va + NBPDR)) - return (KERN_PROTECTION_FAILURE); pde = pmap_pde(pmap, va); if (pde == NULL) { mp = _pmap_allocpte(pmap, pmap_pdpe_pindex(va), NULL, va); - if (mp == NULL) { - if ((flags & PMAP_ENTER_NOSLEEP) != 0) - return (KERN_RESOURCE_SHORTAGE); - PMAP_UNLOCK(pmap); - vm_wait(NULL); - PMAP_LOCK(pmap); - goto restart; - } + if (mp == NULL) + goto allocf; pde = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mp)); pde = &pde[pmap_pde_index(va)]; origpte = *pde; MPASS(origpte == 0); } else { - pdpe = pmap_pdpe(pmap, va); - MPASS(pdpe != NULL && (*pdpe & PG_V) != 0); origpte = *pde; if ((origpte & PG_V) == 0) { + pdpe = pmap_pdpe(pmap, va); + MPASS(pdpe != NULL && (*pdpe & PG_V) != 0); mp = PHYS_TO_VM_PAGE(*pdpe & PG_FRAME); mp->ref_count++; } } - KASSERT((origpte & PG_V) == 0 || ((origpte & PG_PS) != 0 && - (origpte & PG_FRAME) == (pten & PG_FRAME)), - ("va %#lx changing 2M phys page pde %#lx pten %#lx", - va, origpte, pten)); - if ((pten & PG_W) != 0 && (origpte & PG_W) == 0) - pmap->pm_stats.wired_count += NBPDR / PAGE_SIZE; - else if ((pten & PG_W) == 0 && (origpte & PG_W) != 0) - pmap->pm_stats.wired_count -= NBPDR / PAGE_SIZE; *pde = pten; } + KASSERT((origpte & PG_V) == 0 || ((origpte & PG_PS) != 0 && + (origpte & PG_PS_FRAME) == (pten & PG_PS_FRAME)), + ("va %#lx changing %s phys page origpte %#lx pten %#lx", + va, psind == 2 ? "1G" : "2M", origpte, pten)); + if ((pten & PG_W) != 0 && (origpte & PG_W) == 0) + pmap->pm_stats.wired_count += pagesizes[psind] / PAGE_SIZE; + else if ((pten & PG_W) == 0 && (origpte & PG_W) != 0) + pmap->pm_stats.wired_count -= pagesizes[psind] / PAGE_SIZE; if ((origpte & PG_V) == 0) pmap_resident_count_inc(pmap, pagesizes[psind] / PAGE_SIZE); return (KERN_SUCCESS); + +allocf: + if ((flags & PMAP_ENTER_NOSLEEP) != 0) + return (KERN_RESOURCE_SHORTAGE); + PMAP_UNLOCK(pmap); + vm_wait(NULL); + PMAP_LOCK(pmap); + goto restart; } /*