upstream/opnsense-src
1
0
Fork 0
mirror of https://github.com/opnsense/src.git synced 2026-06-11 01:30:30 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Do pmap update only on active CPUs.

Browse source 
The pmap update_page/invalidate_page/invalidate_all operations has to be
done only on active cpus. In the simplest case, if the process is not
active on any other CPUs, we can just do the operation on the current CPU.

This change replaces the call to smp_rendezvous() for these operations with
smp_rendezvous_cpus() in case there more than one active CPU, or with a direct
function call if there is just one active CPU.

This change give significant performance increase in fork/exec benchmarks
on XLR/XLS/XLP with 32 cpus.

Reviewed by:	alc
This commit is contained in:
Jayachandran C. 2011-11-17 13:14:59 +00:00
parent 2cda7d0049
commit cca2c4b446
1 changed files with 73 additions and 107 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

180
sys/mips/mips/pmap.c
View file

@ -181,7 +181,6 @@ static pt_entry_t init_pte_prot(vm_offset_t va, vm_page_t m, vm_prot_t prot);
#ifdef SMP
static void pmap_invalidate_page_action(void *arg);
static void pmap_invalidate_all_action(void *arg);
static void pmap_update_page_action(void *arg);
#endif
@ -622,72 +621,84 @@ pmap_init(void)
* Low level helper routines.....
***************************************************/
#ifdef SMP
static __inline void
pmap_invalidate_all_local(pmap_t pmap)
pmap_call_on_active_cpus(pmap_t pmap, void (*fn)(void *), void *arg)
{
u_int cpuid;
int cpuid, cpu, self;
cpuset_t active_cpus;
cpuid = PCPU_GET(cpuid);
if (pmap == kernel_pmap) {
tlb_invalidate_all();
return;
sched_pin();
if (is_kernel_pmap(pmap)) {
smp_rendezvous(NULL, fn, NULL, arg);
goto out;
}
/* Force ASID update on inactive CPUs */
CPU_FOREACH(cpu) {
if (!CPU_ISSET(cpu, &pmap->pm_active))
pmap->pm_asid[cpu].gen = 0;
}
if (CPU_ISSET(cpuid, &pmap->pm_active))
tlb_invalidate_all_user(pmap);
else
pmap->pm_asid[cpuid].gen = 0;
}
#ifdef SMP
static void
pmap_invalidate_all(pmap_t pmap)
{
smp_rendezvous(0, pmap_invalidate_all_action, 0, pmap);
}
static void
pmap_invalidate_all_action(void *arg)
{
pmap_invalidate_all_local((pmap_t)arg);
}
#else
static void
pmap_invalidate_all(pmap_t pmap)
{
pmap_invalidate_all_local(pmap);
}
#endif
static __inline void
pmap_invalidate_page_local(pmap_t pmap, vm_offset_t va)
{
u_int cpuid;
cpuid = PCPU_GET(cpuid);
/*
* XXX: barrier/locking for active?
*
* Take a snapshot of active here, any further changes are ignored.
* tlb update/invalidate should be harmless on inactive CPUs
*/
active_cpus = pmap->pm_active;
self = CPU_ISSET(cpuid, &active_cpus);
CPU_CLR(cpuid, &active_cpus);
/* Optimize for the case where this cpu is the only active one */
if (CPU_EMPTY(&active_cpus)) {
if (self)
fn(arg);
} else {
if (self)
CPU_SET(cpuid, &active_cpus);
smp_rendezvous_cpus(active_cpus, NULL, fn, NULL, arg);
}
out:
sched_unpin();
}
#else /* !SMP */
static __inline void
pmap_call_on_active_cpus(pmap_t pmap, void (*fn)(void *), void *arg)
{
int cpuid;
if (is_kernel_pmap(pmap)) {
tlb_invalidate_address(pmap, va);
fn(arg);
return;
}
if (pmap->pm_asid[cpuid].gen != PCPU_GET(asid_generation))
return;
else if (!CPU_ISSET(cpuid, &pmap->pm_active)) {
cpuid = PCPU_GET(cpuid);
if (!CPU_ISSET(cpuid, &pmap->pm_active))
pmap->pm_asid[cpuid].gen = 0;
return;
}
tlb_invalidate_address(pmap, va);
else
fn(arg);
}
#endif /* SMP */
static void
pmap_invalidate_all(pmap_t pmap)
{
pmap_call_on_active_cpus(pmap,
(void (*)(void *))tlb_invalidate_all_user, pmap);
}
#ifdef SMP
struct pmap_invalidate_page_arg {
pmap_t pmap;
vm_offset_t va;
};
static void
pmap_invalidate_page_action(void *arg)
{
struct pmap_invalidate_page_arg *p = arg;
tlb_invalidate_address(p->pmap, p->va);
}
static void
pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
{
@ -695,52 +706,23 @@ pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
arg.pmap = pmap;
arg.va = va;
smp_rendezvous(0, pmap_invalidate_page_action, 0, &arg);
pmap_call_on_active_cpus(pmap, pmap_invalidate_page_action, &arg);
}
static void
pmap_invalidate_page_action(void *arg)
{
struct pmap_invalidate_page_arg *p = arg;
pmap_invalidate_page_local(p->pmap, p->va);
}
#else
static void
pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
{
pmap_invalidate_page_local(pmap, va);
}
#endif
static __inline void
pmap_update_page_local(pmap_t pmap, vm_offset_t va, pt_entry_t pte)
{
u_int cpuid;
cpuid = PCPU_GET(cpuid);
if (is_kernel_pmap(pmap)) {
tlb_update(pmap, va, pte);
return;
}
if (pmap->pm_asid[cpuid].gen != PCPU_GET(asid_generation))
return;
else if (!CPU_ISSET(cpuid, &pmap->pm_active)) {
pmap->pm_asid[cpuid].gen = 0;
return;
}
tlb_update(pmap, va, pte);
}
#ifdef SMP
struct pmap_update_page_arg {
pmap_t pmap;
vm_offset_t va;
pt_entry_t pte;
};
static void
pmap_update_page_action(void *arg)
{
struct pmap_update_page_arg *p = arg;
tlb_update(p->pmap, p->va, p->pte);
}
static void
pmap_update_page(pmap_t pmap, vm_offset_t va, pt_entry_t pte)
{
@ -749,25 +731,9 @@ pmap_update_page(pmap_t pmap, vm_offset_t va, pt_entry_t pte)
arg.pmap = pmap;
arg.va = va;
arg.pte = pte;
smp_rendezvous(0, pmap_update_page_action, 0, &arg);
pmap_call_on_active_cpus(pmap, pmap_update_page_action, &arg);
}
static void
pmap_update_page_action(void *arg)
{
struct pmap_update_page_arg *p = arg;
pmap_update_page_local(p->pmap, p->va, p->pte);
}
#else
static void
pmap_update_page(pmap_t pmap, vm_offset_t va, pt_entry_t pte)
{
pmap_update_page_local(pmap, va, pte);
}
#endif
/*
* Routine: pmap_extract
* Function:
@ -3213,7 +3179,7 @@ pmap_emulate_modified(pmap_t pmap, vm_offset_t va)
#ifdef SMP
/* It is possible that some other CPU changed m-bit */
if (!pte_test(pte, PTE_V) || pte_test(pte, PTE_D)) {
pmap_update_page_local(pmap, va, *pte);
tlb_update(pmap, va, *pte);
PMAP_UNLOCK(pmap);
return (0);
}
@ -3227,7 +3193,7 @@ pmap_emulate_modified(pmap_t pmap, vm_offset_t va)
return (1);
}
pte_set(pte, PTE_D);
pmap_update_page_local(pmap, va, *pte);
tlb_update(pmap, va, *pte);
pa = TLBLO_PTE_TO_PA(*pte);
if (!page_is_managed(pa))
panic("pmap_emulate_modified: unmanaged page");