upstream/opnsense-src
1
0
Fork 0
mirror of https://github.com/opnsense/src.git synced 2026-05-28 04:12:45 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

sys: Consolidate common implementation details of PV entries.

Browse source 
Add a <sys/_pv_entry.h> intended for use in <machine/pmap.h> to
define struct pv_entry, pv_chunk, and related macros and inline
functions.

Note that powerpc does not yet use this as while the mmu_radix pmap
in powerpc uses the new scheme (albeit with fewer PV entries in a
chunk than normal due to an used pv_pmap field in struct pv_entry),
the Book-E pmaps for powerpc use the older style PV entries without
chunks (and thus require the pv_pmap field).

Suggested by:	kib
Reviewed by:	kib
Sponsored by:	DARPA
Differential Revision:	https://reviews.freebsd.org/D36685
This commit is contained in:
John Baldwin 2022-10-07 10:14:03 -07:00
parent 61ab88d873
commit 4d90a5afc5
11 changed files with 169 additions and 282 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
sys/amd64/amd64/pmap.c
View file

@ -5092,24 +5092,10 @@ pmap_growkernel(vm_offset_t addr)
* page management routines.
***************************************************/
CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE);
CTASSERT(_NPCM == 3);
CTASSERT(_NPCPV == 168);
static __inline struct pv_chunk *
pv_to_chunk(pv_entry_t pv)
{
return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK));
}
#define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap)
#define PC_FREE0 0xfffffffffffffffful
#define PC_FREE1 0xfffffffffffffffful
#define PC_FREE2 ((1ul << (_NPCPV % 64)) - 1)
static const uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1, PC_FREE2 };
static const uint64_t pc_freemask[_NPCM] = {
[0 ... _NPCM - 2] = PC_FREEN,
[_NPCM - 1] = PC_FREEL
};
#ifdef PV_STATS
@ -5321,8 +5307,7 @@ reclaim_pv_chunk_domain(pmap_t locked_pmap, struct rwlock **lockp, int domain)
PV_STAT(counter_u64_add(pv_entry_spare, freed));
PV_STAT(counter_u64_add(pv_entry_count, -freed));
TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
if (pc->pc_map[0] == PC_FREE0 && pc->pc_map[1] == PC_FREE1 &&
pc->pc_map[2] == PC_FREE2) {
if (pc_is_free(pc)) {
PV_STAT(counter_u64_add(pv_entry_spare, -_NPCPV));
PV_STAT(counter_u64_add(pc_chunk_count, -1));
PV_STAT(counter_u64_add(pc_chunk_frees, 1));
@ -5406,8 +5391,7 @@ free_pv_entry(pmap_t pmap, pv_entry_t pv)
field = idx / 64;
bit = idx % 64;
pc->pc_map[field] |= 1ul << bit;
if (pc->pc_map[0] != PC_FREE0 || pc->pc_map[1] != PC_FREE1 ||
pc->pc_map[2] != PC_FREE2) {
if (!pc_is_free(pc)) {
/* 98% of the time, pc is already at the head of the list. */
if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) {
TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
@ -5532,9 +5516,9 @@ retry:
dump_add_page(m->phys_addr);
pc = (void *)PHYS_TO_DMAP(m->phys_addr);
pc->pc_pmap = pmap;
pc->pc_map[0] = PC_FREE0 & ~1ul; /* preallocated bit 0 */
pc->pc_map[1] = PC_FREE1;
pc->pc_map[2] = PC_FREE2;
pc->pc_map[0] = PC_FREEN & ~1ul; /* preallocated bit 0 */
pc->pc_map[1] = PC_FREEN;
pc->pc_map[2] = PC_FREEL;
pvc = &pv_chunks[vm_page_domain(m)];
mtx_lock(&pvc->pvc_lock);
TAILQ_INSERT_TAIL(&pvc->pvc_list, pc, pc_lru);
@ -5632,9 +5616,9 @@ retry:
dump_add_page(m->phys_addr);
pc = (void *)PHYS_TO_DMAP(m->phys_addr);
pc->pc_pmap = pmap;
pc->pc_map[0] = PC_FREE0;
pc->pc_map[1] = PC_FREE1;
pc->pc_map[2] = PC_FREE2;
pc->pc_map[0] = PC_FREEN;
pc->pc_map[1] = PC_FREEN;
pc->pc_map[2] = PC_FREEL;
TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
TAILQ_INSERT_TAIL(&new_tail[vm_page_domain(m)], pc, pc_lru);
PV_STAT(counter_u64_add(pv_entry_spare, _NPCPV));

37
sys/amd64/include/pmap.h
View file

@ -291,6 +291,7 @@
#include <sys/_lock.h>
#include <sys/_mutex.h>
#include <sys/_pctrie.h>
#include <sys/_pv_entry.h>
#include <sys/_rangeset.h>
#include <sys/_smr.h>
@ -353,8 +354,6 @@ extern pt_entry_t pg_nx;
/*
* Pmap stuff
*/
struct pv_entry;
struct pv_chunk;
/*
* Locks
@ -424,40 +423,6 @@ extern struct pmap kernel_pmap_store;
int pmap_pinit_type(pmap_t pmap, enum pmap_type pm_type, int flags);
int pmap_emulate_accessed_dirty(pmap_t pmap, vm_offset_t va, int ftype);
#endif
/*
* For each vm_page_t, there is a list of all currently valid virtual
* mappings of that page. An entry is a pv_entry_t, the list is pv_list.
*/
typedef struct pv_entry {
vm_offset_t pv_va; /* virtual address for mapping */
TAILQ_ENTRY(pv_entry) pv_next;
} *pv_entry_t;
/*
* pv_entries are allocated in chunks per-process. This avoids the
* need to track per-pmap assignments.
*/
#define _NPCPV 168
#define _NPCM howmany(_NPCPV, 64)
#define PV_CHUNK_HEADER \
pmap_t pc_pmap; \
TAILQ_ENTRY(pv_chunk) pc_list; \
uint64_t pc_map[_NPCM]; /* bitmap; 1 = free */ \
TAILQ_ENTRY(pv_chunk) pc_lru;
struct pv_chunk_header {
PV_CHUNK_HEADER
};
struct pv_chunk {
PV_CHUNK_HEADER
struct pv_entry pc_pventry[_NPCPV];
};
#ifdef _KERNEL
extern caddr_t CADDR1;
extern pt_entry_t *CMAP1;

22
sys/arm/arm/pmap-v6.c
View file

@ -2739,27 +2739,9 @@ pmap_unuse_pt2(pmap_t pmap, vm_offset_t va, struct spglist *free)
*
*************************************/
CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE);
CTASSERT(_NPCM == 11);
CTASSERT(_NPCPV == 336);
static __inline struct pv_chunk *
pv_to_chunk(pv_entry_t pv)
{
return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK));
}
#define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap)
#define PC_FREE0_9 0xfffffffful /* Free values for index 0 through 9 */
#define PC_FREE10 ((1ul << (_NPCPV % 32)) - 1) /* Free values for index 10 */
static const uint32_t pc_freemask[_NPCM] = {
PC_FREE0_9, PC_FREE0_9, PC_FREE0_9,
PC_FREE0_9, PC_FREE0_9, PC_FREE0_9,
PC_FREE0_9, PC_FREE0_9, PC_FREE0_9,
PC_FREE0_9, PC_FREE10
[0 ... _NPCM - 2] = PC_FREEN,
[_NPCM - 1] = PC_FREEL
};
SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, &pv_entry_count, 0,

30
sys/arm/include/pmap-v6.h
View file

@ -52,6 +52,7 @@
#include <sys/_cpuset.h>
#include <sys/_lock.h>
#include <sys/_mutex.h>
#include <sys/_pv_entry.h>
typedef uint32_t pt1_entry_t; /* L1 table entry */
typedef uint32_t pt2_entry_t; /* L2 table entry */
@ -93,9 +94,6 @@ typedef uint32_t ttb_entry_t; /* TTB entry */
/*
* Pmap stuff
*/
struct pv_entry;
struct pv_chunk;
struct md_page {
TAILQ_HEAD(,pv_entry) pv_list;
uint16_t pt2_wirecount[4];
@ -128,33 +126,7 @@ extern struct pmap kernel_pmap_store;
#define PMAP_MTX(pmap) (&(pmap)->pm_mtx)
#define PMAP_TRYLOCK(pmap) mtx_trylock(&(pmap)->pm_mtx)
#define PMAP_UNLOCK(pmap) mtx_unlock(&(pmap)->pm_mtx)
#endif
/*
* For each vm_page_t, there is a list of all currently valid virtual
* mappings of that page. An entry is a pv_entry_t, the list is pv_list.
*/
typedef struct pv_entry {
vm_offset_t pv_va; /* virtual address for mapping */
TAILQ_ENTRY(pv_entry) pv_next;
} *pv_entry_t;
/*
* pv_entries are allocated in chunks per-process. This avoids the
* need to track per-pmap assignments.
*/
#define _NPCPV 336
#define _NPCM howmany(_NPCPV, 32)
struct pv_chunk {
pmap_t pc_pmap;
TAILQ_ENTRY(pv_chunk) pc_list;
uint32_t pc_map[_NPCM]; /* bitmap; 1 = free */
TAILQ_ENTRY(pv_chunk) pc_lru;
struct pv_entry pc_pventry[_NPCPV];
};
#ifdef _KERNEL
extern ttb_entry_t pmap_kern_ttb; /* TTB for kernel pmap */
#define pmap_page_get_memattr(m) ((m)->md.pat_mode)

32
sys/arm64/arm64/pmap.c
View file

@ -2501,43 +2501,11 @@ pmap_growkernel(vm_offset_t addr)
* page management routines.
***************************************************/
CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE);
static __inline struct pv_chunk *
pv_to_chunk(pv_entry_t pv)
{
return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK));
}
#define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap)
#define PC_FREEN 0xfffffffffffffffful
#define PC_FREEL ((1ul << (_NPCPV % 64)) - 1)
static const uint64_t pc_freemask[_NPCM] = {
[0 ... _NPCM - 2] = PC_FREEN,
[_NPCM - 1] = PC_FREEL
};
static __inline bool
pc_is_full(struct pv_chunk *pc)
{
for (u_int i = 0; i < _NPCM; i++)
if (pc->pc_map[i] != 0)
return (false);
return (true);
}
static __inline bool
pc_is_free(struct pv_chunk *pc)
{
for (u_int i = 0; i < _NPCM - 1; i++)
if (pc->pc_map[i] != PC_FREEN)
return (false);
return (pc->pc_map[_NPCM - 1] == PC_FREEL);
}
#ifdef PV_STATS
static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail;

37
sys/arm64/include/pmap.h
View file

@ -43,6 +43,7 @@
#include <sys/queue.h>
#include <sys/_lock.h>
#include <sys/_mutex.h>
#include <sys/_pv_entry.h>
#include <vm/_vm_radix.h>
@ -97,42 +98,6 @@ struct pmap {
};
typedef struct pmap *pmap_t;
typedef struct pv_entry {
vm_offset_t pv_va; /* virtual address for mapping */
TAILQ_ENTRY(pv_entry) pv_next;
} *pv_entry_t;
/*
* pv_entries are allocated in chunks per-process. This avoids the
* need to track per-pmap assignments.
*/
#if PAGE_SIZE == PAGE_SIZE_4K
#define _NPCPV 168
#define _NPAD 0
#elif PAGE_SIZE == PAGE_SIZE_16K
#define _NPCPV 677
#define _NPAD 1
#else
#error Unsupported page size
#endif
#define _NPCM howmany(_NPCPV, 64)
#define PV_CHUNK_HEADER \
pmap_t pc_pmap; \
TAILQ_ENTRY(pv_chunk) pc_list; \
uint64_t pc_map[_NPCM]; /* bitmap; 1 = free */ \
TAILQ_ENTRY(pv_chunk) pc_lru;
struct pv_chunk_header {
PV_CHUNK_HEADER
};
struct pv_chunk {
PV_CHUNK_HEADER
struct pv_entry pc_pventry[_NPCPV];
uint64_t pc_pad[_NPAD];
};
struct thread;
#ifdef _KERNEL

22
sys/i386/i386/pmap.c
View file

@ -2287,27 +2287,9 @@ __CONCAT(PMTYPE, growkernel)(vm_offset_t addr)
* page management routines.
***************************************************/
CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE);
CTASSERT(_NPCM == 11);
CTASSERT(_NPCPV == 336);
static __inline struct pv_chunk *
pv_to_chunk(pv_entry_t pv)
{
return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK));
}
#define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap)
#define PC_FREE0_9 0xfffffffful /* Free values for index 0 through 9 */
#define PC_FREE10 ((1ul << (_NPCPV % 32)) - 1) /* Free values for index 10 */
static const uint32_t pc_freemask[_NPCM] = {
PC_FREE0_9, PC_FREE0_9, PC_FREE0_9,
PC_FREE0_9, PC_FREE0_9, PC_FREE0_9,
PC_FREE0_9, PC_FREE0_9, PC_FREE0_9,
PC_FREE0_9, PC_FREE10
[0 ... _NPCM - 2] = PC_FREEN,
[_NPCM - 1] = PC_FREEL
};
#ifdef PV_STATS

31
sys/i386/include/pmap.h
View file

@ -133,6 +133,7 @@
#include <sys/_cpuset.h>
#include <sys/_lock.h>
#include <sys/_mutex.h>
#include <sys/_pv_entry.h>
#include <vm/_vm_radix.h>
@ -157,9 +158,6 @@
/*
* Pmap stuff
*/
struct pv_entry;
struct pv_chunk;
struct md_page {
TAILQ_HEAD(,pv_entry) pv_list;
int pat_mode;
@ -194,33 +192,6 @@ extern struct pmap kernel_pmap_store;
#define PMAP_MTX(pmap) (&(pmap)->pm_mtx)
#define PMAP_TRYLOCK(pmap) mtx_trylock(&(pmap)->pm_mtx)
#define PMAP_UNLOCK(pmap) mtx_unlock(&(pmap)->pm_mtx)
#endif
/*
* For each vm_page_t, there is a list of all currently valid virtual
* mappings of that page. An entry is a pv_entry_t, the list is pv_list.
*/
typedef struct pv_entry {
vm_offset_t pv_va; /* virtual address for mapping */
TAILQ_ENTRY(pv_entry) pv_next;
} *pv_entry_t;
/*
* pv_entries are allocated in chunks per-process. This avoids the
* need to track per-pmap assignments.
*/
#define _NPCPV 336
#define _NPCM howmany(_NPCPV, 32)
struct pv_chunk {
pmap_t pc_pmap;
TAILQ_ENTRY(pv_chunk) pc_list;
uint32_t pc_map[_NPCM]; /* bitmap; 1 = free */
TAILQ_ENTRY(pv_chunk) pc_lru;
struct pv_entry pc_pventry[_NPCPV];
};
#ifdef _KERNEL
extern char *ptvmmap; /* poor name! */
extern vm_offset_t virtual_avail;

21
sys/riscv/include/pmap.h
View file

@ -44,6 +44,7 @@
#include <sys/_cpuset.h>
#include <sys/_lock.h>
#include <sys/_mutex.h>
#include <sys/_pv_entry.h>
#include <vm/_vm_radix.h>
@ -88,26 +89,6 @@ struct pmap {
struct vm_radix pm_root;
};
typedef struct pv_entry {
vm_offset_t pv_va; /* virtual address for mapping */
TAILQ_ENTRY(pv_entry) pv_next;
} *pv_entry_t;
/*
* pv_entries are allocated in chunks per-process. This avoids the
* need to track per-pmap assignments.
*/
#define _NPCPV 168
#define _NPCM howmany(_NPCPV, 64)
struct pv_chunk {
struct pmap * pc_pmap;
TAILQ_ENTRY(pv_chunk) pc_list;
uint64_t pc_map[_NPCM]; /* bitmap; 1 = free */
TAILQ_ENTRY(pv_chunk) pc_lru;
struct pv_entry pc_pventry[_NPCPV];
};
typedef struct pmap *pmap_t;
#ifdef _KERNEL

45
sys/riscv/riscv/pmap.c
View file

@ -1710,24 +1710,10 @@ pmap_growkernel(vm_offset_t addr)
* page management routines.
***************************************************/
CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE);
CTASSERT(_NPCM == 3);
CTASSERT(_NPCPV == 168);
static __inline struct pv_chunk *
pv_to_chunk(pv_entry_t pv)
{
return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK));
}
#define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap)
#define PC_FREE0 0xfffffffffffffffful
#define PC_FREE1 0xfffffffffffffffful
#define PC_FREE2 ((1ul << (_NPCPV % 64)) - 1)
static const uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1, PC_FREE2 };
static const uint64_t pc_freemask[_NPCM] = {
[0 ... _NPCM - 2] = PC_FREEN,
[_NPCM - 1] = PC_FREEL
};
#if 0
#ifdef PV_STATS
@ -1793,8 +1779,7 @@ free_pv_entry(pmap_t pmap, pv_entry_t pv)
field = idx / 64;
bit = idx % 64;
pc->pc_map[field] |= 1ul << bit;
if (pc->pc_map[0] != PC_FREE0 || pc->pc_map[1] != PC_FREE1 ||
pc->pc_map[2] != PC_FREE2) {
if (!pc_is_free(pc)) {
/* 98% of the time, pc is already at the head of the list. */
if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) {
TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
@ -1856,8 +1841,7 @@ retry:
pv = &pc->pc_pventry[field * 64 + bit];
pc->pc_map[field] &= ~(1ul << bit);
/* If this was the last item, move it to tail */
if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0 &&
pc->pc_map[2] == 0) {
if (pc_is_full(pc)) {
TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc,
pc_list);
@ -1883,9 +1867,9 @@ retry:
dump_add_page(m->phys_addr);
pc = (void *)PHYS_TO_DMAP(m->phys_addr);
pc->pc_pmap = pmap;
pc->pc_map[0] = PC_FREE0 & ~1ul; /* preallocated bit 0 */
pc->pc_map[1] = PC_FREE1;
pc->pc_map[2] = PC_FREE2;
pc->pc_map[0] = PC_FREEN & ~1ul; /* preallocated bit 0 */
pc->pc_map[1] = PC_FREEN;
pc->pc_map[2] = PC_FREEL;
mtx_lock(&pv_chunks_mutex);
TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru);
mtx_unlock(&pv_chunks_mutex);
@ -1947,9 +1931,9 @@ retry:
#endif
pc = (void *)PHYS_TO_DMAP(m->phys_addr);
pc->pc_pmap = pmap;
pc->pc_map[0] = PC_FREE0;
pc->pc_map[1] = PC_FREE1;
pc->pc_map[2] = PC_FREE2;
pc->pc_map[0] = PC_FREEN;
pc->pc_map[1] = PC_FREEN;
pc->pc_map[2] = PC_FREEL;
TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
TAILQ_INSERT_TAIL(&new_tail, pc, pc_lru);
@ -2065,8 +2049,7 @@ pmap_pv_demote_l2(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
va_last = va + L2_SIZE - PAGE_SIZE;
for (;;) {
pc = TAILQ_FIRST(&pmap->pm_pvchunk);
KASSERT(pc->pc_map[0] != 0 || pc->pc_map[1] != 0 ||
pc->pc_map[2] != 0, ("pmap_pv_demote_l2: missing spare"));
KASSERT(!pc_is_full(pc), ("pmap_pv_demote_l2: missing spare"));
for (field = 0; field < _NPCM; field++) {
while (pc->pc_map[field] != 0) {
bit = ffsl(pc->pc_map[field]) - 1;
@ -2087,7 +2070,7 @@ pmap_pv_demote_l2(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
}
out:
if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0 && pc->pc_map[2] == 0) {
if (pc_is_free(pc)) {
TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
}

134
sys/sys/_pv_entry.h Normal file
View file

@ -0,0 +1,134 @@
/*-
* SPDX-License-Identifier: BSD-3-Clause
*
* Copyright (c) 2003 Peter Wemm.
* Copyright (c) 1991 Regents of the University of California.
* All rights reserved.
*
* This code is derived from software contributed to Berkeley by
* the Systems Programming Group of the University of Utah Computer
* Science Department and William Jolitz of UUNET Technologies Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef __SYS__PV_ENTRY_H__
#define __SYS__PV_ENTRY_H__
struct pmap;
/*
* For each vm_page_t, there is a list of all currently valid virtual
* mappings of that page. An entry is a pv_entry_t, the list is pv_list.
*/
typedef struct pv_entry {
vm_offset_t pv_va; /* virtual address for mapping */
TAILQ_ENTRY(pv_entry) pv_next;
} *pv_entry_t;
/*
* pv_entries are allocated in chunks per-process. This avoids the
* need to track per-pmap assignments. Each chunk is the size of a
* single page.
*
* Chunks store a bitmap in pc_map[] to track which entries in the
* bitmap are free (1) or used (0). PC_FREEL is the value of the last
* entry in the pc_map[] array when a chunk is completely free. PC_FREEN
* is the value of all the other entries in the pc_map[] array when a
* chunk is completely free.
*/
#if PAGE_SIZE == 4 * 1024
#ifdef __LP64__
#define _NPCPV 168
#define _NPAD 0
#else
#define _NPCPV 336
#define _NPAD 0
#endif
#elif PAGE_SIZE == 16 * 1024
#ifdef __LP64__
#define _NPCPV 677
#define _NPAD 1
#endif
#endif
#ifndef _NPCPV
#error Unsupported page size
#endif
#define _NPCM howmany(_NPCPV, __LONG_WIDTH__)
#define PC_FREEN ~0ul
#define PC_FREEL ((1ul << (_NPCPV % __LONG_WIDTH__)) - 1)
#define PV_CHUNK_HEADER \
struct pmap *pc_pmap; \
TAILQ_ENTRY(pv_chunk) pc_list; \
unsigned long pc_map[_NPCM]; /* bitmap; 1 = free */ \
TAILQ_ENTRY(pv_chunk) pc_lru;
struct pv_chunk_header {
PV_CHUNK_HEADER
};
struct pv_chunk {
PV_CHUNK_HEADER
struct pv_entry pc_pventry[_NPCPV];
unsigned long pc_pad[_NPAD];
};
_Static_assert(sizeof(struct pv_chunk) == PAGE_SIZE,
"PV entry chunk size mismatch");
#ifdef _KERNEL
static __inline bool
pc_is_full(struct pv_chunk *pc)
{
for (u_int i = 0; i < _NPCM; i++) {
if (pc->pc_map[i] != 0)
return (false);
}
return (true);
}
static __inline bool
pc_is_free(struct pv_chunk *pc)
{
for (u_int i = 0; i < _NPCM - 1; i++) {
if (pc->pc_map[i] != PC_FREEN)
return (false);
}
return (pc->pc_map[_NPCM - 1] == PC_FREEL);
}
static __inline struct pv_chunk *
pv_to_chunk(pv_entry_t pv)
{
return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK));
}
#define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap)
#endif
#endif /* !__SYS__PV_ENTRY_H__ */