diff --git a/sys/arm/allwinner/a10/a10_intc.c b/sys/arm/allwinner/a10/a10_intc.c index 410fba4b472..0bac9edbfd0 100644 --- a/sys/arm/allwinner/a10/a10_intc.c +++ b/sys/arm/allwinner/a10/a10_intc.c @@ -236,7 +236,7 @@ a10_intr_pic_attach(struct a10_aintc_softc *sc) if (pic == NULL) return (ENXIO); - return (intr_pic_claim_root(sc->sc_dev, xref, a10_intr, sc)); + return (intr_pic_claim_root(sc->sc_dev, xref, a10_intr, sc, INTR_ROOT_IRQ)); } static void diff --git a/sys/arm/arm/exception.S b/sys/arm/arm/exception.S index 7230830f78c..a17898a53d4 100644 --- a/sys/arm/arm/exception.S +++ b/sys/arm/arm/exception.S @@ -48,6 +48,8 @@ #include "assym.inc" +#include + #include #include #include @@ -308,7 +310,8 @@ ASENTRY_NP(irq_entry) PUSHFRAMEINSVC /* mode stack, build trapframe there. */ adr lr, exception_exit /* Return from handler via standard */ mov r0, sp /* exception exit routine. Pass the */ - b _C_LABEL(intr_irq_handler)/* trapframe to the handler. */ + mov r1, #INTR_ROOT_IRQ /* trapframe and PIC root to the handler. */ + b _C_LABEL(intr_irq_handler) END(irq_entry) /* diff --git a/sys/arm/arm/gic.c b/sys/arm/arm/gic.c index 7fbf7e7fd82..b1b7aacd63a 100644 --- a/sys/arm/arm/gic.c +++ b/sys/arm/arm/gic.c @@ -200,7 +200,7 @@ gic_cpu_mask(struct arm_gic_softc *sc) #ifdef SMP static void -arm_gic_init_secondary(device_t dev) +arm_gic_init_secondary(device_t dev, uint32_t rootnum) { struct arm_gic_softc *sc = device_get_softc(dev); u_int irq, cpu; diff --git a/sys/arm/arm/gic_acpi.c b/sys/arm/arm/gic_acpi.c index fa823320dcc..6fd6f527137 100644 --- a/sys/arm/arm/gic_acpi.c +++ b/sys/arm/arm/gic_acpi.c @@ -227,7 +227,7 @@ gic_acpi_attach(device_t dev) /* * Controller is root: */ - if (intr_pic_claim_root(dev, xref, arm_gic_intr, sc) != 0) { + if (intr_pic_claim_root(dev, xref, arm_gic_intr, sc, INTR_ROOT_IRQ) != 0) { device_printf(dev, "could not set PIC as a root\n"); intr_pic_deregister(dev, xref); goto cleanup; diff --git a/sys/arm/arm/gic_fdt.c b/sys/arm/arm/gic_fdt.c index 7ebc4faaa86..06a869eaae4 100644 --- a/sys/arm/arm/gic_fdt.c +++ b/sys/arm/arm/gic_fdt.c @@ -153,7 +153,8 @@ gic_fdt_attach(device_t dev) */ pxref = ofw_bus_find_iparent(ofw_bus_get_node(dev)); if (pxref == 0 || xref == pxref) { - if (intr_pic_claim_root(dev, xref, arm_gic_intr, sc) != 0) { + if (intr_pic_claim_root(dev, xref, arm_gic_intr, sc, INTR_ROOT_IRQ) + != 0) { device_printf(dev, "could not set PIC as a root\n"); intr_pic_deregister(dev, xref); goto cleanup; diff --git a/sys/arm/broadcom/bcm2835/bcm2835_intr.c b/sys/arm/broadcom/bcm2835/bcm2835_intr.c index 5a8457bc4b6..65ada57c0b6 100644 --- a/sys/arm/broadcom/bcm2835/bcm2835_intr.c +++ b/sys/arm/broadcom/bcm2835/bcm2835_intr.c @@ -405,7 +405,8 @@ bcm_intc_attach(device_t dev) sc->intc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (sc->intc_irq_res == NULL) { - if (intr_pic_claim_root(dev, xref, bcm2835_intc_intr, sc) != 0) { + if (intr_pic_claim_root(dev, xref, bcm2835_intc_intr, sc, INTR_ROOT_IRQ) + != 0) { /* XXX clean up */ device_printf(dev, "could not set PIC as a root\n"); return (ENXIO); diff --git a/sys/arm/broadcom/bcm2835/bcm2836.c b/sys/arm/broadcom/bcm2835/bcm2836.c index 8ff824d3452..7ed9dedaa77 100644 --- a/sys/arm/broadcom/bcm2835/bcm2836.c +++ b/sys/arm/broadcom/bcm2835/bcm2836.c @@ -538,7 +538,7 @@ bcm_lintc_init_pmu_on_ap(struct bcm_lintc_softc *sc, u_int cpu) } static void -bcm_lintc_init_secondary(device_t dev) +bcm_lintc_init_secondary(device_t dev, uint32_t rootnum) { u_int cpu; struct bcm_lintc_softc *sc; @@ -646,7 +646,8 @@ bcm_lintc_pic_attach(struct bcm_lintc_softc *sc) if (pic == NULL) return (ENXIO); - error = intr_pic_claim_root(sc->bls_dev, xref, bcm_lintc_intr, sc); + error = intr_pic_claim_root(sc->bls_dev, xref, bcm_lintc_intr, sc, + INTR_ROOT_IRQ); if (error != 0) return (error); diff --git a/sys/arm/ti/aintc.c b/sys/arm/ti/aintc.c index 776d231c3c4..b822a203821 100644 --- a/sys/arm/ti/aintc.c +++ b/sys/arm/ti/aintc.c @@ -230,7 +230,8 @@ ti_aintc_pic_attach(struct ti_aintc_softc *sc) if (pic == NULL) return (ENXIO); - return (intr_pic_claim_root(sc->sc_dev, xref, ti_aintc_intr, sc)); + return (intr_pic_claim_root(sc->sc_dev, xref, ti_aintc_intr, sc, + INTR_ROOT_IRQ)); } static int diff --git a/sys/arm64/arm64/exception.S b/sys/arm64/arm64/exception.S index 3dff834e27a..9e33fd99850 100644 --- a/sys/arm64/arm64/exception.S +++ b/sys/arm64/arm64/exception.S @@ -26,6 +26,7 @@ */ #include +#include #include #include @@ -232,6 +233,7 @@ ENTRY(handle_el1h_irq) save_registers 1 KMSAN_ENTER mov x0, sp + mov x1, #INTR_ROOT_IRQ bl intr_irq_handler KMSAN_LEAVE restore_registers 1 @@ -266,6 +268,7 @@ ENTRY(handle_el0_irq) save_registers 0 KMSAN_ENTER mov x0, sp + mov x1, #INTR_ROOT_IRQ bl intr_irq_handler do_ast KMSAN_LEAVE diff --git a/sys/arm64/arm64/gic_v3.c b/sys/arm64/arm64/gic_v3.c index 0b25650b3fb..964a129111e 100644 --- a/sys/arm64/arm64/gic_v3.c +++ b/sys/arm64/arm64/gic_v3.c @@ -1093,7 +1093,7 @@ gic_v3_bind_intr(device_t dev, struct intr_irqsrc *isrc) #ifdef SMP static void -gic_v3_init_secondary(device_t dev) +gic_v3_init_secondary(device_t dev, uint32_t rootnum) { struct gic_v3_setup_periph_args pargs; device_t child; @@ -1140,7 +1140,7 @@ gic_v3_init_secondary(device_t dev) for (i = 0; i < sc->gic_nchildren; i++) { child = sc->gic_children[i]; - PIC_INIT_SECONDARY(child); + PIC_INIT_SECONDARY(child, rootnum); } } diff --git a/sys/arm64/arm64/gic_v3_acpi.c b/sys/arm64/arm64/gic_v3_acpi.c index 180066b8893..86ddb316da1 100644 --- a/sys/arm64/arm64/gic_v3_acpi.c +++ b/sys/arm64/arm64/gic_v3_acpi.c @@ -345,8 +345,9 @@ gic_v3_acpi_attach(device_t dev) } } - if (intr_pic_claim_root(dev, ACPI_INTR_XREF, arm_gic_v3_intr, sc) - != 0) { + err = intr_pic_claim_root(dev, ACPI_INTR_XREF, arm_gic_v3_intr, sc, + INTR_ROOT_IRQ); + if (err != 0) { err = ENXIO; goto error; } diff --git a/sys/arm64/arm64/gic_v3_fdt.c b/sys/arm64/arm64/gic_v3_fdt.c index 5637f693395..f913d3bfaee 100644 --- a/sys/arm64/arm64/gic_v3_fdt.c +++ b/sys/arm64/arm64/gic_v3_fdt.c @@ -161,7 +161,8 @@ gic_v3_fdt_attach(device_t dev) /* Register xref */ OF_device_register_xref(xref, dev); - if (intr_pic_claim_root(dev, xref, arm_gic_v3_intr, sc) != 0) { + err = intr_pic_claim_root(dev, xref, arm_gic_v3_intr, sc, INTR_ROOT_IRQ); + if (err != 0) { err = ENXIO; goto error; } diff --git a/sys/arm64/arm64/gicv3_its.c b/sys/arm64/arm64/gicv3_its.c index d69f86023da..a79706526c8 100644 --- a/sys/arm64/arm64/gicv3_its.c +++ b/sys/arm64/arm64/gicv3_its.c @@ -1284,7 +1284,7 @@ gicv3_its_setup_intr(device_t dev, struct intr_irqsrc *isrc, #ifdef SMP static void -gicv3_its_init_secondary(device_t dev) +gicv3_its_init_secondary(device_t dev, uint32_t rootnum) { struct gicv3_its_softc *sc; diff --git a/sys/kern/pic_if.m b/sys/kern/pic_if.m index 53117e4b15f..f78e787594c 100644 --- a/sys/kern/pic_if.m +++ b/sys/kern/pic_if.m @@ -74,7 +74,7 @@ CODE { } static void - null_pic_init_secondary(device_t dev) + null_pic_init_secondary(device_t dev, uint32_t rootnum) { } @@ -157,6 +157,7 @@ METHOD void pre_ithread { METHOD void init_secondary { device_t dev; + uint32_t rootnum; } DEFAULT null_pic_init_secondary; METHOD void ipi_send { diff --git a/sys/kern/subr_intr.c b/sys/kern/subr_intr.c index 21098986841..a6052f28b04 100644 --- a/sys/kern/subr_intr.c +++ b/sys/kern/subr_intr.c @@ -89,6 +89,15 @@ #define INTRNAME_LEN (2*MAXCOMLEN + 1) +/* + * Archs may define multiple roots with INTR_ROOT_NUM to support different kinds + * of interrupts (e.g. arm64 FIQs which use a different exception vector than + * IRQs). + */ +#if !defined(INTR_ROOT_NUM) +#define INTR_ROOT_NUM 1 +#endif + #ifdef DEBUG #define debugf(fmt, args...) do { printf("%s(): ", __func__); \ printf(fmt,##args); } while (0) @@ -99,13 +108,14 @@ MALLOC_DECLARE(M_INTRNG); MALLOC_DEFINE(M_INTRNG, "intr", "intr interrupt handling"); -/* Main interrupt handler called from assembler -> 'hidden' for C code. */ -void intr_irq_handler(struct trapframe *tf); - /* Root interrupt controller stuff. */ -device_t intr_irq_root_dev; -static intr_irq_filter_t *irq_root_filter; -static void *irq_root_arg; +struct intr_irq_root { + device_t dev; + intr_irq_filter_t *filter; + void *arg; +}; + +static struct intr_irq_root intr_irq_roots[INTR_ROOT_NUM]; struct intr_pic_child { SLIST_ENTRY(intr_pic_child) pc_next; @@ -327,12 +337,17 @@ isrc_release_counters(struct intr_irqsrc *isrc) * from the assembler, where CPU interrupt is served. */ void -intr_irq_handler(struct trapframe *tf) +intr_irq_handler(struct trapframe *tf, uint32_t rootnum) { struct trapframe * oldframe; struct thread * td; + struct intr_irq_root *root; - KASSERT(irq_root_filter != NULL, ("%s: no filter", __func__)); + KASSERT(rootnum < INTR_ROOT_NUM, + ("%s: invalid interrupt root %d", __func__, rootnum)); + + root = &intr_irq_roots[rootnum]; + KASSERT(root->filter != NULL, ("%s: no filter", __func__)); kasan_mark(tf, sizeof(*tf), sizeof(*tf), 0); kmsan_mark(tf, sizeof(*tf), KMSAN_STATE_INITED); @@ -342,7 +357,7 @@ intr_irq_handler(struct trapframe *tf) td = curthread; oldframe = td->td_intr_frame; td->td_intr_frame = tf; - irq_root_filter(irq_root_arg); + (root->filter)(root->arg); td->td_intr_frame = oldframe; critical_exit(); #ifdef HWPMC_HOOKS @@ -479,6 +494,14 @@ isrc_free_irq(struct intr_irqsrc *isrc) return (0); } +device_t +intr_irq_root_device(uint32_t rootnum) +{ + KASSERT(rootnum < INTR_ROOT_NUM, + ("%s: invalid interrupt root %d", __func__, rootnum)); + return (intr_irq_roots[rootnum].dev); +} + /* * Initialize interrupt source and register it into global interrupt table. */ @@ -877,9 +900,10 @@ intr_pic_deregister(device_t dev, intptr_t xref) */ int intr_pic_claim_root(device_t dev, intptr_t xref, intr_irq_filter_t *filter, - void *arg) + void *arg, uint32_t rootnum) { struct intr_pic *pic; + struct intr_irq_root *root; pic = pic_lookup(dev, xref, FLAG_PIC); if (pic == NULL) { @@ -901,14 +925,17 @@ intr_pic_claim_root(device_t dev, intptr_t xref, intr_irq_filter_t *filter, * Note that we further suppose that there is not threaded interrupt * routine (handler) on the root. See intr_irq_handler(). */ - if (intr_irq_root_dev != NULL) { + KASSERT(rootnum < INTR_ROOT_NUM, + ("%s: invalid interrupt root %d", __func__, rootnum)); + root = &intr_irq_roots[rootnum]; + if (root->dev != NULL) { device_printf(dev, "another root already set\n"); return (EBUSY); } - intr_irq_root_dev = dev; - irq_root_filter = filter; - irq_root_arg = arg; + root->dev = dev; + root->filter = filter; + root->arg = arg; debugf("irq root set to %s\n", device_get_nameunit(dev)); return (0); @@ -1552,14 +1579,19 @@ dosoftints(void) void intr_pic_init_secondary(void) { + device_t dev; + uint32_t rootnum; /* - * QQQ: Only root PIC is aware of other CPUs ??? + * QQQ: Only root PICs are aware of other CPUs ??? */ - KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__)); - //mtx_lock(&isrc_table_lock); - PIC_INIT_SECONDARY(intr_irq_root_dev); + for (rootnum = 0; rootnum < INTR_ROOT_NUM; rootnum++) { + dev = intr_irq_roots[rootnum].dev; + if (dev != NULL) { + PIC_INIT_SECONDARY(dev, rootnum); + } + } //mtx_unlock(&isrc_table_lock); } #endif diff --git a/sys/riscv/riscv/aplic.c b/sys/riscv/riscv/aplic.c index 19b80409c0f..af58bb01474 100644 --- a/sys/riscv/riscv/aplic.c +++ b/sys/riscv/riscv/aplic.c @@ -321,6 +321,7 @@ aplic_setup_direct_mode(device_t dev) int error = ENXIO; u_int irq; int cpu, hartid, rid, i, nintr, idc; + device_t rootdev; sc = device_get_softc(dev); node = ofw_bus_get_node(dev); @@ -407,7 +408,8 @@ aplic_setup_direct_mode(device_t dev) APLIC_IDC_ITHRESHOLD_DISABLE); } - iparent = OF_xref_from_node(ofw_bus_get_node(intr_irq_root_dev)); + rootdev = intr_irq_root_device(INTR_ROOT_IRQ); + iparent = OF_xref_from_node(ofw_bus_get_node(rootdev)); cell = IRQ_EXTERNAL_SUPERVISOR; irq = ofw_bus_map_intr(dev, iparent, 1, &cell); error = bus_set_resource(dev, SYS_RES_IRQ, 0, irq, 1); diff --git a/sys/riscv/riscv/intc.c b/sys/riscv/riscv/intc.c index 399bb05bbcf..248175e8bea 100644 --- a/sys/riscv/riscv/intc.c +++ b/sys/riscv/riscv/intc.c @@ -181,7 +181,7 @@ intc_attach(device_t dev) if (pic == NULL) return (ENXIO); - return (intr_pic_claim_root(sc->dev, xref, intc_intr, sc)); + return (intr_pic_claim_root(sc->dev, xref, intc_intr, sc, INTR_ROOT_IRQ)); } static void @@ -241,7 +241,7 @@ intc_setup_intr(device_t dev, struct intr_irqsrc *isrc, #ifdef SMP static void -intc_init_secondary(device_t dev) +intc_init_secondary(device_t dev, uint32_t rootnum) { struct intc_softc *sc; struct intr_irqsrc *isrc; diff --git a/sys/riscv/riscv/sbi_ipi.c b/sys/riscv/riscv/sbi_ipi.c index d694b476f5f..fac5c0c39b9 100644 --- a/sys/riscv/riscv/sbi_ipi.c +++ b/sys/riscv/riscv/sbi_ipi.c @@ -141,6 +141,7 @@ sbi_ipi_attach(device_t dev) int irq, rid, error; phandle_t iparent; pcell_t cell; + device_t rootdev; sc = device_get_softc(dev); sc->dev = dev; @@ -155,7 +156,8 @@ sbi_ipi_attach(device_t dev) return (ENXIO); } - iparent = OF_xref_from_node(ofw_bus_get_node(intr_irq_root_dev)); + rootdev = intr_irq_root_device(INTR_ROOT_IRQ); + iparent = OF_xref_from_node(ofw_bus_get_node(rootdev)); cell = IRQ_SOFTWARE_SUPERVISOR; irq = ofw_bus_map_intr(dev, iparent, 1, &cell); error = bus_set_resource(dev, SYS_RES_IRQ, 0, irq, 1); diff --git a/sys/riscv/riscv/timer.c b/sys/riscv/riscv/timer.c index 456dd463ab2..7ff8a84f376 100644 --- a/sys/riscv/riscv/timer.c +++ b/sys/riscv/riscv/timer.c @@ -196,6 +196,7 @@ riscv_timer_attach(device_t dev) int irq, rid, error; phandle_t iparent; pcell_t cell; + device_t rootdev; sc = device_get_softc(dev); if (riscv_timer_sc != NULL) @@ -211,7 +212,8 @@ riscv_timer_attach(device_t dev) riscv_timer_sc = sc; - iparent = OF_xref_from_node(ofw_bus_get_node(intr_irq_root_dev)); + rootdev = intr_irq_root_device(INTR_ROOT_IRQ); + iparent = OF_xref_from_node(ofw_bus_get_node(rootdev)); cell = IRQ_TIMER_SUPERVISOR; irq = ofw_bus_map_intr(dev, iparent, 1, &cell); error = bus_set_resource(dev, SYS_RES_IRQ, 0, irq, 1); diff --git a/sys/riscv/riscv/trap.c b/sys/riscv/riscv/trap.c index 3a7b3fba26b..89eb6a1a378 100644 --- a/sys/riscv/riscv/trap.c +++ b/sys/riscv/riscv/trap.c @@ -74,8 +74,6 @@ #include #endif -void intr_irq_handler(struct trapframe *tf); - int (*dtrace_invop_jump_addr)(struct trapframe *); /* Called from exception.S */ @@ -319,7 +317,7 @@ do_trap_supervisor(struct trapframe *frame) exception = frame->tf_scause & SCAUSE_CODE; if ((frame->tf_scause & SCAUSE_INTR) != 0) { /* Interrupt */ - intr_irq_handler(frame); + intr_irq_handler(frame, INTR_ROOT_IRQ); return; } @@ -400,7 +398,7 @@ do_trap_user(struct trapframe *frame) exception = frame->tf_scause & SCAUSE_CODE; if ((frame->tf_scause & SCAUSE_INTR) != 0) { /* Interrupt */ - intr_irq_handler(frame); + intr_irq_handler(frame, INTR_ROOT_IRQ); return; } intr_enable(); diff --git a/sys/sys/intr.h b/sys/sys/intr.h index f6957864c63..54e838e5015 100644 --- a/sys/sys/intr.h +++ b/sys/sys/intr.h @@ -33,10 +33,15 @@ #error Need INTRNG for this file #endif +#ifndef LOCORE #include +#endif #define INTR_IRQ_INVALID 0xFFFFFFFF +#define INTR_ROOT_IRQ 0 + +#ifndef LOCORE enum intr_map_data_type { INTR_MAP_DATA_ACPI = 0, INTR_MAP_DATA_FDT, @@ -110,12 +115,13 @@ u_int intr_irq_next_cpu(u_int current_cpu, cpuset_t *cpumask); struct intr_pic *intr_pic_register(device_t, intptr_t); int intr_pic_deregister(device_t, intptr_t); -int intr_pic_claim_root(device_t, intptr_t, intr_irq_filter_t *, void *); +int intr_pic_claim_root(device_t, intptr_t, intr_irq_filter_t *, void *, + uint32_t); int intr_pic_add_handler(device_t, struct intr_pic *, intr_child_irq_filter_t *, void *, uintptr_t, uintptr_t); bool intr_is_per_cpu(struct resource *); -extern device_t intr_irq_root_dev; +device_t intr_irq_root_device(uint32_t); /* Intr interface for BUS. */ @@ -163,4 +169,8 @@ void intr_ipi_send(cpuset_t cpus, u_int ipi); void intr_ipi_dispatch(u_int ipi); #endif +/* Main interrupt handler called from asm on most archs except riscv. */ +void intr_irq_handler(struct trapframe *tf, uint32_t rootnum); + +#endif /* !LOCORE */ #endif /* _SYS_INTR_H */