upstream/opnsense-src
1
0
Fork 0
mirror of https://github.com/opnsense/src.git synced 2026-04-21 22:27:47 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions
opnsense-src/sys/dev/hwpmc/hwpmc_core.c
Mitchell Horne 5adb91c729 pmc: better distinguish pmu-events allocation path 
Background:

The pm_ev field of struct pmc_op_pmcallocate and struct pmc
traditionally contains the index of the chosen event, corresponding to
the __PMC_EVENTS array in pmc_events.h. This is a static list of events,
maintained by FreeBSD.

In the usual case, libpmc translates the user supplied event name
(string) into the pm_ev index, which is passed as an argument to the
allocation syscall. On the kernel side, the allocation method for the
relevant hwpmc class translates the given index into the event code that
will be written to an event selection register.

In 2018, a new source of performance event definitions was introduced:
the pmu-events json files, which are maintained by the Linux kernel. The
result was better coverage for newer Intel processors with a reduced
maintenance burden for libpmc/hwpmc. Intel and AMD CPUs were
unconditionally switched to allocate events from pmu-events instead of
the traditional scheme (959826ca1b, 81eb4dcf9e).

Under the pmu-events scheme, the pm_ev field contains an index
corresponding to the selected event from the pmu-events table, something
which the kernel has no knowledge of. The configuration for the
performance counting registers is instead passed via class-dependent
fields (struct pmc_md_op_pmcallocate).

In 2021 I changed the allocation logic so that it would attempt to
pull from the pmu-events table first, and fall-back to the traditional
method (dfb4fb4116). Later, pmu-events support for arm64 and power8
CPUs was added (28dd6730a5 and b48a2770d4).

The problem that remains is that the pm_ev field is overloaded, without
a definitive way to determine whether the event allocation came from the
pmu-events table or FreeBSD's statically-defined PMC events. This
resulted in a recent fix, 21f7397a61.

Change:

To disambiguate these two supported but separate use-cases, add a new
flag, PMC_F_EV_PMU, to be set as part of the allocation, indicating that
the event index came from pmu-events.

This is useful in two ways:
 1. On the kernel side, we can validate the syscall arguments better.
    Some classes support only the traditional event scheme (e.g.
    hwpmc_armv7), while others support only the pmu-events method (e.g.
    hwpmc_core for Intel). We can now check for this. The hwpmc_arm64
    class supports both methods, so the new flag supersedes the existing
    MD flag, PM_MD_EVENT_RAW.

 2. The flag will be tracked in struct pmc for the duration of its
    lifetime, meaning it is communicated back to userspace. This allows
    libpmc to perform the reverse index-to-event-name translation
    without speculating about the meaning of the index value.

Adding the flag is a backwards-incompatible ABI change. We recently
bumped the major version of the hwpmc module, so this breakage is
acceptable.

Reviewed by:	jkoshy
MFC after:	3 days
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D40753

(cherry picked from commit c190fb35f35cc163b61e582a49115680b0d49dcc)
2023-09-11 12:19:58 -07:00

1265 lines
30 KiB
C
Raw Permalink Blame History

/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2008 Joseph Koshy
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
/*
* Intel Core PMCs.
*/
#include <sys/cdefs.h>
#include <sys/param.h>
#include <sys/bus.h>
#include <sys/pmc.h>
#include <sys/pmckern.h>
#include <sys/smp.h>
#include <sys/systm.h>
#include <machine/intr_machdep.h>
#include <x86/apicvar.h>
#include <machine/cpu.h>
#include <machine/cpufunc.h>
#include <machine/md_var.h>
#include <machine/specialreg.h>
#define CORE_CPUID_REQUEST 0xA
#define CORE_CPUID_REQUEST_SIZE 0x4
#define CORE_CPUID_EAX 0x0
#define CORE_CPUID_EBX 0x1
#define CORE_CPUID_ECX 0x2
#define CORE_CPUID_EDX 0x3
#define IAF_PMC_CAPS \
(PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INTERRUPT | \
PMC_CAP_USER | PMC_CAP_SYSTEM)
#define IAF_RI_TO_MSR(RI) ((RI) + (1 << 30))
#define IAP_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | PMC_CAP_SYSTEM | \
PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE | \
PMC_CAP_INVERT | PMC_CAP_QUALIFIER | PMC_CAP_PRECISE)
#define EV_IS_NOTARCH 0
#define EV_IS_ARCH_SUPP 1
#define EV_IS_ARCH_NOTSUPP -1
/*
* "Architectural" events defined by Intel. The values of these
* symbols correspond to positions in the bitmask returned by
* the CPUID.0AH instruction.
*/
enum core_arch_events {
CORE_AE_BRANCH_INSTRUCTION_RETIRED = 5,
CORE_AE_BRANCH_MISSES_RETIRED = 6,
CORE_AE_INSTRUCTION_RETIRED = 1,
CORE_AE_LLC_MISSES = 4,
CORE_AE_LLC_REFERENCE = 3,
CORE_AE_UNHALTED_REFERENCE_CYCLES = 2,
CORE_AE_UNHALTED_CORE_CYCLES = 0
};
static enum pmc_cputype core_cputype;
static int core_version;
struct core_cpu {
volatile uint32_t pc_iafctrl; /* Fixed function control. */
volatile uint64_t pc_globalctrl; /* Global control register. */
struct pmc_hw pc_corepmcs[];
};
static struct core_cpu **core_pcpu;
static uint32_t core_architectural_events;
static uint64_t core_pmcmask;
static int core_iaf_ri; /* relative index of fixed counters */
static int core_iaf_width;
static int core_iaf_npmc;
static int core_iap_width;
static int core_iap_npmc;
static int core_iap_wroffset;
static u_int pmc_alloc_refs;
static bool pmc_tsx_force_abort_set;
static int
core_pcpu_noop(struct pmc_mdep *md, int cpu)
{
(void) md;
(void) cpu;
return (0);
}
static int
core_pcpu_init(struct pmc_mdep *md, int cpu)
{
struct pmc_cpu *pc;
struct core_cpu *cc;
struct pmc_hw *phw;
int core_ri, n, npmc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[iaf,%d] insane cpu number %d", __LINE__, cpu));
PMCDBG1(MDP,INI,1,"core-init cpu=%d", cpu);
core_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_ri;
npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_num;
if (core_version >= 2)
npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF].pcd_num;
cc = malloc(sizeof(struct core_cpu) + npmc * sizeof(struct pmc_hw),
M_PMC, M_WAITOK | M_ZERO);
core_pcpu[cpu] = cc;
pc = pmc_pcpu[cpu];
KASSERT(pc != NULL && cc != NULL,
("[core,%d] NULL per-cpu structures cpu=%d", __LINE__, cpu));
for (n = 0, phw = cc->pc_corepmcs; n < npmc; n++, phw++) {
phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
PMC_PHW_CPU_TO_STATE(cpu) |
PMC_PHW_INDEX_TO_STATE(n + core_ri);
phw->phw_pmc = NULL;
pc->pc_hwpmcs[n + core_ri] = phw;
}
if (core_version >= 2 && vm_guest == VM_GUEST_NO) {
/* Enable Freezing PMCs on PMI. */
wrmsr(MSR_DEBUGCTLMSR, rdmsr(MSR_DEBUGCTLMSR) | 0x1000);
}
return (0);
}
static int
core_pcpu_fini(struct pmc_mdep *md, int cpu)
{
int core_ri, n, npmc;
struct pmc_cpu *pc;
struct core_cpu *cc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[core,%d] insane cpu number (%d)", __LINE__, cpu));
PMCDBG1(MDP,INI,1,"core-pcpu-fini cpu=%d", cpu);
if ((cc = core_pcpu[cpu]) == NULL)
return (0);
core_pcpu[cpu] = NULL;
pc = pmc_pcpu[cpu];
KASSERT(pc != NULL, ("[core,%d] NULL per-cpu %d state", __LINE__,
cpu));
npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_num;
core_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_ri;
for (n = 0; n < npmc; n++)
wrmsr(IAP_EVSEL0 + n, 0);
if (core_version >= 2) {
wrmsr(IAF_CTRL, 0);
npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF].pcd_num;
}
for (n = 0; n < npmc; n++)
pc->pc_hwpmcs[n + core_ri] = NULL;
free(cc, M_PMC);
return (0);
}
/*
* Fixed function counters.
*/
static pmc_value_t
iaf_perfctr_value_to_reload_count(pmc_value_t v)
{
/* If the PMC has overflowed, return a reload count of zero. */
if ((v & (1ULL << (core_iaf_width - 1))) == 0)
return (0);
v &= (1ULL << core_iaf_width) - 1;
return (1ULL << core_iaf_width) - v;
}
static pmc_value_t
iaf_reload_count_to_perfctr_value(pmc_value_t rlc)
{
return (1ULL << core_iaf_width) - rlc;
}
static int
iaf_allocate_pmc(int cpu, int ri, struct pmc *pm,
const struct pmc_op_pmcallocate *a)
{
uint8_t ev, umask;
uint32_t caps;
uint64_t config, flags;
const struct pmc_md_iap_op_pmcallocate *iap;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[core,%d] illegal CPU %d", __LINE__, cpu));
PMCDBG2(MDP,ALL,1, "iaf-allocate ri=%d reqcaps=0x%x", ri, pm->pm_caps);
if (ri < 0 || ri > core_iaf_npmc)
return (EINVAL);
if (a->pm_class != PMC_CLASS_IAF)
return (EINVAL);
if ((a->pm_flags & PMC_F_EV_PMU) == 0)
return (EINVAL);
iap = &a->pm_md.pm_iap;
config = iap->pm_iap_config;
ev = IAP_EVSEL_GET(config);
umask = IAP_UMASK_GET(config);
if (ev == 0x0) {
if (umask != ri + 1)
return (EINVAL);
} else {
switch (ri) {
case 0: /* INST_RETIRED.ANY */
if (ev != 0xC0 || umask != 0x00)
return (EINVAL);
break;
case 1: /* CPU_CLK_UNHALTED.THREAD */
if (ev != 0x3C || umask != 0x00)
return (EINVAL);
break;
case 2: /* CPU_CLK_UNHALTED.REF */
if (ev != 0x3C || umask != 0x01)
return (EINVAL);
break;
case 3: /* TOPDOWN.SLOTS */
if (ev != 0xA4 || umask != 0x01)
return (EINVAL);
break;
default:
return (EINVAL);
}
}
pmc_alloc_refs++;
if ((cpu_stdext_feature3 & CPUID_STDEXT3_TSXFA) != 0 &&
!pmc_tsx_force_abort_set) {
pmc_tsx_force_abort_set = true;
x86_msr_op(MSR_TSX_FORCE_ABORT, MSR_OP_RENDEZVOUS_ALL |
MSR_OP_WRITE, 1, NULL);
}
flags = 0;
if (config & IAP_OS)
flags |= IAF_OS;
if (config & IAP_USR)
flags |= IAF_USR;
if (config & IAP_ANY)
flags |= IAF_ANY;
if (config & IAP_INT)
flags |= IAF_PMI;
caps = a->pm_caps;
if (caps & PMC_CAP_INTERRUPT)
flags |= IAF_PMI;
if (caps & PMC_CAP_SYSTEM)
flags |= IAF_OS;
if (caps & PMC_CAP_USER)
flags |= IAF_USR;
if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0)
flags |= (IAF_OS | IAF_USR);
pm->pm_md.pm_iaf.pm_iaf_ctrl = (flags << (ri * 4));
PMCDBG1(MDP,ALL,2, "iaf-allocate config=0x%jx",
(uintmax_t) pm->pm_md.pm_iaf.pm_iaf_ctrl);
return (0);
}
static int
iaf_config_pmc(int cpu, int ri, struct pmc *pm)
{
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[core,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < core_iaf_npmc,
("[core,%d] illegal row-index %d", __LINE__, ri));
PMCDBG3(MDP,CFG,1, "iaf-config cpu=%d ri=%d pm=%p", cpu, ri, pm);
KASSERT(core_pcpu[cpu] != NULL, ("[core,%d] null per-cpu %d", __LINE__,
cpu));
core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc = pm;
return (0);
}
static int
iaf_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
{
struct pmc_hw *phw;
phw = &core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri];
snprintf(pi->pm_name, sizeof(pi->pm_name), "IAF-%d", ri);
pi->pm_class = PMC_CLASS_IAF;
if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
pi->pm_enabled = TRUE;
*ppmc = phw->phw_pmc;
} else {
pi->pm_enabled = FALSE;
*ppmc = NULL;
}
return (0);
}
static int
iaf_get_config(int cpu, int ri, struct pmc **ppm)
{
*ppm = core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc;
return (0);
}
static int
iaf_get_msr(int ri, uint32_t *msr)
{
KASSERT(ri >= 0 && ri < core_iaf_npmc,
("[iaf,%d] ri %d out of range", __LINE__, ri));
*msr = IAF_RI_TO_MSR(ri);
return (0);
}
static int
iaf_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v)
{
pmc_value_t tmp;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[core,%d] illegal cpu value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < core_iaf_npmc,
("[core,%d] illegal row-index %d", __LINE__, ri));
tmp = rdpmc(IAF_RI_TO_MSR(ri));
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
*v = iaf_perfctr_value_to_reload_count(tmp);
else
*v = tmp & ((1ULL << core_iaf_width) - 1);
PMCDBG4(MDP,REA,1, "iaf-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,
IAF_RI_TO_MSR(ri), *v);
return (0);
}
static int
iaf_release_pmc(int cpu, int ri, struct pmc *pmc)
{
PMCDBG3(MDP,REL,1, "iaf-release cpu=%d ri=%d pm=%p", cpu, ri, pmc);
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[core,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < core_iaf_npmc,
("[core,%d] illegal row-index %d", __LINE__, ri));
KASSERT(core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc == NULL,
("[core,%d] PHW pmc non-NULL", __LINE__));
MPASS(pmc_alloc_refs > 0);
if (pmc_alloc_refs-- == 1 && pmc_tsx_force_abort_set) {
pmc_tsx_force_abort_set = false;
x86_msr_op(MSR_TSX_FORCE_ABORT, MSR_OP_RENDEZVOUS_ALL |
MSR_OP_WRITE, 0, NULL);
}
return (0);
}
static int
iaf_start_pmc(int cpu, int ri, struct pmc *pm)
{
struct core_cpu *cc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[core,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < core_iaf_npmc,
("[core,%d] illegal row-index %d", __LINE__, ri));
PMCDBG2(MDP,STA,1,"iaf-start cpu=%d ri=%d", cpu, ri);
cc = core_pcpu[cpu];
cc->pc_iafctrl |= pm->pm_md.pm_iaf.pm_iaf_ctrl;
wrmsr(IAF_CTRL, cc->pc_iafctrl);
cc->pc_globalctrl |= (1ULL << (ri + IAF_OFFSET));
wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl);
PMCDBG4(MDP,STA,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)",
cc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL),
cc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL));
return (0);
}
static int
iaf_stop_pmc(int cpu, int ri, struct pmc *pm)
{
struct core_cpu *cc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[core,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < core_iaf_npmc,
("[core,%d] illegal row-index %d", __LINE__, ri));
PMCDBG2(MDP,STA,1,"iaf-stop cpu=%d ri=%d", cpu, ri);
cc = core_pcpu[cpu];
cc->pc_iafctrl &= ~(IAF_MASK << (ri * 4));
wrmsr(IAF_CTRL, cc->pc_iafctrl);
/* Don't need to write IA_GLOBAL_CTRL, one disable is enough. */
PMCDBG4(MDP,STO,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)",
cc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL),
cc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL));
return (0);
}
static int
iaf_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v)
{
struct core_cpu *cc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[core,%d] illegal cpu value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < core_iaf_npmc,
("[core,%d] illegal row-index %d", __LINE__, ri));
cc = core_pcpu[cpu];
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
v = iaf_reload_count_to_perfctr_value(v);
/* Turn off the fixed counter */
wrmsr(IAF_CTRL, cc->pc_iafctrl & ~(IAF_MASK << (ri * 4)));
wrmsr(IAF_CTR0 + ri, v & ((1ULL << core_iaf_width) - 1));
/* Turn on fixed counters */
wrmsr(IAF_CTRL, cc->pc_iafctrl);
PMCDBG6(MDP,WRI,1, "iaf-write cpu=%d ri=%d msr=0x%x v=%jx iafctrl=%jx "
"pmc=%jx", cpu, ri, IAF_RI_TO_MSR(ri), v,
(uintmax_t) rdmsr(IAF_CTRL),
(uintmax_t) rdpmc(IAF_RI_TO_MSR(ri)));
return (0);
}
static void
iaf_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth)
{
struct pmc_classdep *pcd;
KASSERT(md != NULL, ("[iaf,%d] md is NULL", __LINE__));
PMCDBG0(MDP,INI,1, "iaf-initialize");
pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF];
pcd->pcd_caps = IAF_PMC_CAPS;
pcd->pcd_class = PMC_CLASS_IAF;
pcd->pcd_num = npmc;
pcd->pcd_ri = md->pmd_npmc;
pcd->pcd_width = pmcwidth;
pcd->pcd_allocate_pmc = iaf_allocate_pmc;
pcd->pcd_config_pmc = iaf_config_pmc;
pcd->pcd_describe = iaf_describe;
pcd->pcd_get_config = iaf_get_config;
pcd->pcd_get_msr = iaf_get_msr;
pcd->pcd_pcpu_fini = core_pcpu_noop;
pcd->pcd_pcpu_init = core_pcpu_noop;
pcd->pcd_read_pmc = iaf_read_pmc;
pcd->pcd_release_pmc = iaf_release_pmc;
pcd->pcd_start_pmc = iaf_start_pmc;
pcd->pcd_stop_pmc = iaf_stop_pmc;
pcd->pcd_write_pmc = iaf_write_pmc;
md->pmd_npmc += npmc;
}
/*
* Intel programmable PMCs.
*/
/* Sub fields of UMASK that this event supports. */
#define IAP_M_CORE (1 << 0) /* Core specificity */
#define IAP_M_AGENT (1 << 1) /* Agent specificity */
#define IAP_M_PREFETCH (1 << 2) /* Prefetch */
#define IAP_M_MESI (1 << 3) /* MESI */
#define IAP_M_SNOOPRESPONSE (1 << 4) /* Snoop response */
#define IAP_M_SNOOPTYPE (1 << 5) /* Snoop type */
#define IAP_M_TRANSITION (1 << 6) /* Transition */
#define IAP_F_CORE (0x3 << 14) /* Core specificity */
#define IAP_F_AGENT (0x1 << 13) /* Agent specificity */
#define IAP_F_PREFETCH (0x3 << 12) /* Prefetch */
#define IAP_F_MESI (0xF << 8) /* MESI */
#define IAP_F_SNOOPRESPONSE (0xB << 8) /* Snoop response */
#define IAP_F_SNOOPTYPE (0x3 << 8) /* Snoop type */
#define IAP_F_TRANSITION (0x1 << 12) /* Transition */
#define IAP_PREFETCH_RESERVED (0x2 << 12)
#define IAP_CORE_THIS (0x1 << 14)
#define IAP_CORE_ALL (0x3 << 14)
#define IAP_F_CMASK 0xFF000000
static pmc_value_t
iap_perfctr_value_to_reload_count(pmc_value_t v)
{
/* If the PMC has overflowed, return a reload count of zero. */
if ((v & (1ULL << (core_iap_width - 1))) == 0)
return (0);
v &= (1ULL << core_iap_width) - 1;
return (1ULL << core_iap_width) - v;
}
static pmc_value_t
iap_reload_count_to_perfctr_value(pmc_value_t rlc)
{
return (1ULL << core_iap_width) - rlc;
}
static int
iap_pmc_has_overflowed(int ri)
{
uint64_t v;
/*
* We treat a Core (i.e., Intel architecture v1) PMC as has
* having overflowed if its MSB is zero.
*/
v = rdpmc(ri);
return ((v & (1ULL << (core_iap_width - 1))) == 0);
}
static int
iap_event_corei7_ok_on_counter(uint8_t evsel, int ri)
{
uint32_t mask;
switch (evsel) {
/* Events valid only on counter 0, 1. */
case 0x40:
case 0x41:
case 0x42:
case 0x43:
case 0x4C:
case 0x4E:
case 0x51:
case 0x52:
case 0x53:
case 0x63:
mask = 0x3;
break;
/* Any row index is ok. */
default:
mask = ~0;
}
return (mask & (1 << ri));
}
static int
iap_event_westmere_ok_on_counter(uint8_t evsel, int ri)
{
uint32_t mask;
switch (evsel) {
/* Events valid only on counter 0. */
case 0x60:
case 0xB3:
mask = 0x1;
break;
/* Events valid only on counter 0, 1. */
case 0x4C:
case 0x4E:
case 0x51:
case 0x52:
case 0x63:
mask = 0x3;
break;
/* Any row index is ok. */
default:
mask = ~0;
}
return (mask & (1 << ri));
}
static int
iap_event_sb_sbx_ib_ibx_ok_on_counter(uint8_t evsel, int ri)
{
uint32_t mask;
switch (evsel) {
/* Events valid only on counter 0. */
case 0xB7:
mask = 0x1;
break;
/* Events valid only on counter 1. */
case 0xC0:
mask = 0x2;
break;
/* Events valid only on counter 2. */
case 0x48:
case 0xA2:
case 0xA3:
mask = 0x4;
break;
/* Events valid only on counter 3. */
case 0xBB:
case 0xCD:
mask = 0x8;
break;
/* Any row index is ok. */
default:
mask = ~0;
}
return (mask & (1 << ri));
}
static int
iap_event_core_ok_on_counter(uint8_t evsel, int ri)
{
uint32_t mask;
switch (evsel) {
/*
* Events valid only on counter 0.
*/
case 0x10:
case 0x14:
case 0x18:
case 0xB3:
case 0xC1:
case 0xCB:
mask = (1 << 0);
break;
/*
* Events valid only on counter 1.
*/
case 0x11:
case 0x12:
case 0x13:
mask = (1 << 1);
break;
default:
mask = ~0; /* Any row index is ok. */
}
return (mask & (1 << ri));
}
static int
iap_allocate_pmc(int cpu, int ri, struct pmc *pm,
const struct pmc_op_pmcallocate *a)
{
uint8_t ev;
const struct pmc_md_iap_op_pmcallocate *iap;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[core,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < core_iap_npmc,
("[core,%d] illegal row-index value %d", __LINE__, ri));
if (a->pm_class != PMC_CLASS_IAP)
return (EINVAL);
if ((a->pm_flags & PMC_F_EV_PMU) == 0)
return (EINVAL);
iap = &a->pm_md.pm_iap;
ev = IAP_EVSEL_GET(iap->pm_iap_config);
switch (core_cputype) {
case PMC_CPU_INTEL_CORE:
case PMC_CPU_INTEL_CORE2:
case PMC_CPU_INTEL_CORE2EXTREME:
if (iap_event_core_ok_on_counter(ev, ri) == 0)
return (EINVAL);
case PMC_CPU_INTEL_COREI7:
case PMC_CPU_INTEL_NEHALEM_EX:
if (iap_event_corei7_ok_on_counter(ev, ri) == 0)
return (EINVAL);
break;
case PMC_CPU_INTEL_WESTMERE:
case PMC_CPU_INTEL_WESTMERE_EX:
if (iap_event_westmere_ok_on_counter(ev, ri) == 0)
return (EINVAL);
break;
case PMC_CPU_INTEL_SANDYBRIDGE:
case PMC_CPU_INTEL_SANDYBRIDGE_XEON:
case PMC_CPU_INTEL_IVYBRIDGE:
case PMC_CPU_INTEL_IVYBRIDGE_XEON:
case PMC_CPU_INTEL_HASWELL:
case PMC_CPU_INTEL_HASWELL_XEON:
case PMC_CPU_INTEL_BROADWELL:
case PMC_CPU_INTEL_BROADWELL_XEON:
if (iap_event_sb_sbx_ib_ibx_ok_on_counter(ev, ri) == 0)
return (EINVAL);
break;
case PMC_CPU_INTEL_ATOM:
case PMC_CPU_INTEL_ATOM_SILVERMONT:
case PMC_CPU_INTEL_ATOM_GOLDMONT:
case PMC_CPU_INTEL_ATOM_GOLDMONT_P:
case PMC_CPU_INTEL_ATOM_TREMONT:
case PMC_CPU_INTEL_SKYLAKE:
case PMC_CPU_INTEL_SKYLAKE_XEON:
case PMC_CPU_INTEL_ICELAKE:
case PMC_CPU_INTEL_ICELAKE_XEON:
case PMC_CPU_INTEL_ALDERLAKE:
default:
break;
}
pm->pm_md.pm_iap.pm_iap_evsel = iap->pm_iap_config;
return (0);
}
static int
iap_config_pmc(int cpu, int ri, struct pmc *pm)
{
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[core,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < core_iap_npmc,
("[core,%d] illegal row-index %d", __LINE__, ri));
PMCDBG3(MDP,CFG,1, "iap-config cpu=%d ri=%d pm=%p", cpu, ri, pm);
KASSERT(core_pcpu[cpu] != NULL, ("[core,%d] null per-cpu %d", __LINE__,
cpu));
core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc = pm;
return (0);
}
static int
iap_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
{
struct pmc_hw *phw;
phw = &core_pcpu[cpu]->pc_corepmcs[ri];
snprintf(pi->pm_name, sizeof(pi->pm_name), "IAP-%d", ri);
pi->pm_class = PMC_CLASS_IAP;
if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
pi->pm_enabled = TRUE;
*ppmc = phw->phw_pmc;
} else {
pi->pm_enabled = FALSE;
*ppmc = NULL;
}
return (0);
}
static int
iap_get_config(int cpu, int ri, struct pmc **ppm)
{
*ppm = core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc;
return (0);
}
static int
iap_get_msr(int ri, uint32_t *msr)
{
KASSERT(ri >= 0 && ri < core_iap_npmc,
("[iap,%d] ri %d out of range", __LINE__, ri));
*msr = ri;
return (0);
}
static int
iap_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v)
{
pmc_value_t tmp;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[core,%d] illegal cpu value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < core_iap_npmc,
("[core,%d] illegal row-index %d", __LINE__, ri));
tmp = rdpmc(ri);
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
*v = iap_perfctr_value_to_reload_count(tmp);
else
*v = tmp & ((1ULL << core_iap_width) - 1);
PMCDBG4(MDP,REA,1, "iap-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,
IAP_PMC0 + ri, *v);
return (0);
}
static int
iap_release_pmc(int cpu, int ri, struct pmc *pm)
{
(void) pm;
PMCDBG3(MDP,REL,1, "iap-release cpu=%d ri=%d pm=%p", cpu, ri,
pm);
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[core,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < core_iap_npmc,
("[core,%d] illegal row-index %d", __LINE__, ri));
KASSERT(core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc
== NULL, ("[core,%d] PHW pmc non-NULL", __LINE__));
return (0);
}
static int
iap_start_pmc(int cpu, int ri, struct pmc *pm)
{
uint64_t evsel;
struct core_cpu *cc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[core,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < core_iap_npmc,
("[core,%d] illegal row-index %d", __LINE__, ri));
cc = core_pcpu[cpu];
PMCDBG2(MDP,STA,1, "iap-start cpu=%d ri=%d", cpu, ri);
evsel = pm->pm_md.pm_iap.pm_iap_evsel;
PMCDBG4(MDP,STA,2, "iap-start/2 cpu=%d ri=%d evselmsr=0x%x evsel=0x%x",
cpu, ri, IAP_EVSEL0 + ri, evsel);
/* Event specific configuration. */
switch (IAP_EVSEL_GET(evsel)) {
case 0xB7:
wrmsr(IA_OFFCORE_RSP0, pm->pm_md.pm_iap.pm_iap_rsp);
break;
case 0xBB:
wrmsr(IA_OFFCORE_RSP1, pm->pm_md.pm_iap.pm_iap_rsp);
break;
default:
break;
}
wrmsr(IAP_EVSEL0 + ri, evsel | IAP_EN);
if (core_version >= 2) {
cc->pc_globalctrl |= (1ULL << ri);
wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl);
}
return (0);
}
static int
iap_stop_pmc(int cpu, int ri, struct pmc *pm __unused)
{
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[core,%d] illegal cpu value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < core_iap_npmc,
("[core,%d] illegal row index %d", __LINE__, ri));
PMCDBG2(MDP,STO,1, "iap-stop cpu=%d ri=%d", cpu, ri);
wrmsr(IAP_EVSEL0 + ri, 0);
/* Don't need to write IA_GLOBAL_CTRL, one disable is enough. */
return (0);
}
static int
iap_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v)
{
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[core,%d] illegal cpu value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < core_iap_npmc,
("[core,%d] illegal row index %d", __LINE__, ri));
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
v = iap_reload_count_to_perfctr_value(v);
v &= (1ULL << core_iap_width) - 1;
PMCDBG4(MDP,WRI,1, "iap-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri,
IAP_PMC0 + ri, v);
/*
* Write the new value to the counter (or it's alias). The
* counter will be in a stopped state when the pcd_write()
* entry point is called.
*/
wrmsr(core_iap_wroffset + IAP_PMC0 + ri, v);
return (0);
}
static void
iap_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth,
int flags)
{
struct pmc_classdep *pcd;
KASSERT(md != NULL, ("[iap,%d] md is NULL", __LINE__));
PMCDBG0(MDP,INI,1, "iap-initialize");
/* Remember the set of architectural events supported. */
core_architectural_events = ~flags;
pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP];
pcd->pcd_caps = IAP_PMC_CAPS;
pcd->pcd_class = PMC_CLASS_IAP;
pcd->pcd_num = npmc;
pcd->pcd_ri = md->pmd_npmc;
pcd->pcd_width = pmcwidth;
pcd->pcd_allocate_pmc = iap_allocate_pmc;
pcd->pcd_config_pmc = iap_config_pmc;
pcd->pcd_describe = iap_describe;
pcd->pcd_get_config = iap_get_config;
pcd->pcd_get_msr = iap_get_msr;
pcd->pcd_pcpu_fini = core_pcpu_fini;
pcd->pcd_pcpu_init = core_pcpu_init;
pcd->pcd_read_pmc = iap_read_pmc;
pcd->pcd_release_pmc = iap_release_pmc;
pcd->pcd_start_pmc = iap_start_pmc;
pcd->pcd_stop_pmc = iap_stop_pmc;
pcd->pcd_write_pmc = iap_write_pmc;
md->pmd_npmc += npmc;
}
static int
core_intr(struct trapframe *tf)
{
pmc_value_t v;
struct pmc *pm;
struct core_cpu *cc;
int error, found_interrupt, ri;
PMCDBG3(MDP,INT, 1, "cpu=%d tf=%p um=%d", curcpu, (void *) tf,
TRAPF_USERMODE(tf));
found_interrupt = 0;
cc = core_pcpu[curcpu];
for (ri = 0; ri < core_iap_npmc; ri++) {
if ((pm = cc->pc_corepmcs[ri].phw_pmc) == NULL ||
!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
continue;
if (!iap_pmc_has_overflowed(ri))
continue;
found_interrupt = 1;
if (pm->pm_state != PMC_STATE_RUNNING)
continue;
error = pmc_process_interrupt(PMC_HR, pm, tf);
v = pm->pm_sc.pm_reloadcount;
v = iap_reload_count_to_perfctr_value(v);
/*
* Stop the counter, reload it but only restart it if
* the PMC is not stalled.
*/
wrmsr(IAP_EVSEL0 + ri, pm->pm_md.pm_iap.pm_iap_evsel);
wrmsr(core_iap_wroffset + IAP_PMC0 + ri, v);
if (__predict_false(error))
continue;
wrmsr(IAP_EVSEL0 + ri, pm->pm_md.pm_iap.pm_iap_evsel | IAP_EN);
}
if (found_interrupt)
counter_u64_add(pmc_stats.pm_intr_processed, 1);
else
counter_u64_add(pmc_stats.pm_intr_ignored, 1);
if (found_interrupt)
lapic_reenable_pmc();
return (found_interrupt);
}
static int
core2_intr(struct trapframe *tf)
{
int error, found_interrupt = 0, n, cpu;
uint64_t flag, intrstatus, intrdisable = 0;
struct pmc *pm;
struct core_cpu *cc;
pmc_value_t v;
cpu = curcpu;
PMCDBG3(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf,
TRAPF_USERMODE(tf));
/*
* The IA_GLOBAL_STATUS (MSR 0x38E) register indicates which
* PMCs have a pending PMI interrupt. We take a 'snapshot' of
* the current set of interrupting PMCs and process these
* after stopping them.
*/
intrstatus = rdmsr(IA_GLOBAL_STATUS);
PMCDBG2(MDP,INT, 1, "cpu=%d intrstatus=%jx", cpu,
(uintmax_t) intrstatus);
/*
* Stop PMCs unless hardware already done it.
*/
if ((intrstatus & IA_GLOBAL_STATUS_FLAG_CTR_FRZ) == 0)
wrmsr(IA_GLOBAL_CTRL, 0);
cc = core_pcpu[cpu];
KASSERT(cc != NULL, ("[core,%d] null pcpu", __LINE__));
/*
* Look for interrupts from fixed function PMCs.
*/
for (n = 0, flag = (1ULL << IAF_OFFSET); n < core_iaf_npmc;
n++, flag <<= 1) {
if ((intrstatus & flag) == 0)
continue;
found_interrupt = 1;
pm = cc->pc_corepmcs[n + core_iaf_ri].phw_pmc;
if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING ||
!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
continue;
error = pmc_process_interrupt(PMC_HR, pm, tf);
if (__predict_false(error))
intrdisable |= flag;
v = iaf_reload_count_to_perfctr_value(pm->pm_sc.pm_reloadcount);
/* Reload sampling count. */
wrmsr(IAF_CTR0 + n, v);
PMCDBG4(MDP,INT, 1, "iaf-intr cpu=%d error=%d v=%jx(%jx)", curcpu,
error, (uintmax_t) v, (uintmax_t) rdpmc(IAF_RI_TO_MSR(n)));
}
/*
* Process interrupts from the programmable counters.
*/
for (n = 0, flag = 1; n < core_iap_npmc; n++, flag <<= 1) {
if ((intrstatus & flag) == 0)
continue;
found_interrupt = 1;
pm = cc->pc_corepmcs[n].phw_pmc;
if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING ||
!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
continue;
error = pmc_process_interrupt(PMC_HR, pm, tf);
if (__predict_false(error))
intrdisable |= flag;
v = iap_reload_count_to_perfctr_value(pm->pm_sc.pm_reloadcount);
PMCDBG3(MDP,INT, 1, "iap-intr cpu=%d error=%d v=%jx", cpu, error,
(uintmax_t) v);
/* Reload sampling count. */
wrmsr(core_iap_wroffset + IAP_PMC0 + n, v);
}
if (found_interrupt)
counter_u64_add(pmc_stats.pm_intr_processed, 1);
else
counter_u64_add(pmc_stats.pm_intr_ignored, 1);
if (found_interrupt)
lapic_reenable_pmc();
/*
* Reenable all non-stalled PMCs.
*/
if ((intrstatus & IA_GLOBAL_STATUS_FLAG_CTR_FRZ) == 0) {
wrmsr(IA_GLOBAL_OVF_CTRL, intrstatus);
cc->pc_globalctrl &= ~intrdisable;
wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl);
} else {
if (__predict_false(intrdisable)) {
cc->pc_globalctrl &= ~intrdisable;
wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl);
}
wrmsr(IA_GLOBAL_OVF_CTRL, intrstatus);
}
PMCDBG4(MDP, INT, 1, "cpu=%d fixedctrl=%jx globalctrl=%jx status=%jx",
cpu, (uintmax_t) rdmsr(IAF_CTRL),
(uintmax_t) rdmsr(IA_GLOBAL_CTRL),
(uintmax_t) rdmsr(IA_GLOBAL_STATUS));
return (found_interrupt);
}
int
pmc_core_initialize(struct pmc_mdep *md, int maxcpu, int version_override)
{
int cpuid[CORE_CPUID_REQUEST_SIZE];
int flags, nflags;
do_cpuid(CORE_CPUID_REQUEST, cpuid);
core_cputype = md->pmd_cputype;
core_version = (version_override > 0) ? version_override :
cpuid[CORE_CPUID_EAX] & 0xFF;
PMCDBG3(MDP,INI,1,"core-init cputype=%d ncpu=%d version=%d",
core_cputype, maxcpu, core_version);
if (core_version < 1 || core_version > 5 ||
(core_cputype != PMC_CPU_INTEL_CORE && core_version == 1)) {
/* Unknown PMC architecture. */
printf("hwpmc_core: unknown PMC architecture: %d\n",
core_version);
return (EPROGMISMATCH);
}
core_iap_wroffset = 0;
if (cpu_feature2 & CPUID2_PDCM) {
if (rdmsr(IA32_PERF_CAPABILITIES) & PERFCAP_FW_WRITE) {
PMCDBG0(MDP, INI, 1,
"core-init full-width write supported");
core_iap_wroffset = IAP_A_PMC0 - IAP_PMC0;
} else
PMCDBG0(MDP, INI, 1,
"core-init full-width write NOT supported");
} else
PMCDBG0(MDP, INI, 1, "core-init pdcm not supported");
core_pmcmask = 0;
/*
* Initialize programmable counters.
*/
core_iap_npmc = (cpuid[CORE_CPUID_EAX] >> 8) & 0xFF;
core_iap_width = (cpuid[CORE_CPUID_EAX] >> 16) & 0xFF;
core_pmcmask |= ((1ULL << core_iap_npmc) - 1);
nflags = (cpuid[CORE_CPUID_EAX] >> 24) & 0xFF;
flags = cpuid[CORE_CPUID_EBX] & ((1 << nflags) - 1);
iap_initialize(md, maxcpu, core_iap_npmc, core_iap_width, flags);
/*
* Initialize fixed function counters, if present.
*/
if (core_version >= 2) {
core_iaf_ri = core_iap_npmc;
core_iaf_npmc = cpuid[CORE_CPUID_EDX] & 0x1F;
core_iaf_width = (cpuid[CORE_CPUID_EDX] >> 5) & 0xFF;
iaf_initialize(md, maxcpu, core_iaf_npmc, core_iaf_width);
core_pmcmask |= ((1ULL << core_iaf_npmc) - 1) << IAF_OFFSET;
}
PMCDBG2(MDP,INI,1,"core-init pmcmask=0x%jx iafri=%d", core_pmcmask,
core_iaf_ri);
core_pcpu = malloc(sizeof(*core_pcpu) * maxcpu, M_PMC,
M_ZERO | M_WAITOK);
/*
* Choose the appropriate interrupt handler.
*/
if (core_version >= 2)
md->pmd_intr = core2_intr;
else
md->pmd_intr = core_intr;
return (0);
}
void
pmc_core_finalize(struct pmc_mdep *md)
{
PMCDBG0(MDP,INI,1, "core-finalize");
free(core_pcpu, M_PMC);
core_pcpu = NULL;
}
Reference in a new issue View git blame Copy permalink