From 6ec8bf9f3d17b712d657c42f74fba936dd284d04 Mon Sep 17 00:00:00 2001 From: Jessica Clarke Date: Wed, 24 Jan 2024 23:49:54 +0000 Subject: [PATCH] riscv: Convert local interrupt controller to a newbus PIC Currently the local interrupt controller implementation is based on pre-INTRNG arm/arm64 code, using hand-rolled event code rather than INTRNG. This then interacts weirdly with the PLIC, and other future interrupt controllers like the APLIC and IMSICs in the upcoming AIA specification, since they become the root PIC despite not being the logical root. Instead, use a real newbus device for it and register it as the root PIC. This also adapts the IPI code to make use of the newly-added INTRNG generic IPI handling framework, adding a new sbi_ipi as the PIC. In future there will be alternative devices for sending IPIs that will register with higher priorities, such as the proposed AIA IMSIC and ACLINT SSWI. Reviewed by: mhorne MFC after: 1 month Differential Revision: https://reviews.freebsd.org/D35901 --- sys/conf/files.riscv | 3 +- sys/riscv/include/intr.h | 24 +-- sys/riscv/include/smp.h | 17 +- sys/riscv/riscv/intc.c | 311 ++++++++++++++++++++++++++++++++ sys/riscv/riscv/intr_machdep.c | 274 ---------------------------- sys/riscv/riscv/mp_machdep.c | 156 +++++++++------- sys/riscv/riscv/plic.c | 49 +++-- sys/riscv/riscv/riscv_console.c | 2 - sys/riscv/riscv/sbi.c | 43 +++++ sys/riscv/riscv/sbi_ipi.c | 208 +++++++++++++++++++++ sys/riscv/riscv/timer.c | 33 +++- sys/riscv/riscv/trap.c | 6 +- 12 files changed, 732 insertions(+), 394 deletions(-) create mode 100644 sys/riscv/riscv/intc.c delete mode 100644 sys/riscv/riscv/intr_machdep.c create mode 100644 sys/riscv/riscv/sbi_ipi.c diff --git a/sys/conf/files.riscv b/sys/conf/files.riscv index 975f9d3f576..b6a87c63422 100644 --- a/sys/conf/files.riscv +++ b/sys/conf/files.riscv @@ -43,7 +43,7 @@ riscv/riscv/elf_machdep.c standard riscv/riscv/exception.S standard riscv/riscv/exec_machdep.c standard riscv/riscv/gdb_machdep.c optional gdb -riscv/riscv/intr_machdep.c standard +riscv/riscv/intc.c standard riscv/riscv/identcpu.c standard riscv/riscv/locore.S standard no-obj riscv/riscv/machdep.c standard @@ -58,6 +58,7 @@ riscv/riscv/ptrace_machdep.c standard riscv/riscv/riscv_console.c optional rcons riscv/riscv/riscv_syscon.c optional syscon riscv_syscon fdt riscv/riscv/sbi.c standard +riscv/riscv/sbi_ipi.c optional smp riscv/riscv/stack_machdep.c optional ddb | stack riscv/riscv/support.S standard riscv/riscv/swtch.S standard diff --git a/sys/riscv/include/intr.h b/sys/riscv/include/intr.h index a4e3a709349..ad811dcbc44 100644 --- a/sys/riscv/include/intr.h +++ b/sys/riscv/include/intr.h @@ -35,32 +35,11 @@ #ifndef _MACHINE_INTR_MACHDEP_H_ #define _MACHINE_INTR_MACHDEP_H_ -#define RISCV_NIRQ 1024 - #ifndef NIRQ -#define NIRQ RISCV_NIRQ +#define NIRQ 1024 #endif -#ifdef INTRNG #include -#endif - -struct trapframe; - -int riscv_teardown_intr(void *); -int riscv_setup_intr(const char *, driver_filter_t *, driver_intr_t *, - void *, int, int, void **); -void riscv_cpu_intr(struct trapframe *); - -typedef unsigned long * riscv_intrcnt_t; - -riscv_intrcnt_t riscv_intrcnt_create(const char *); -void riscv_intrcnt_setname(riscv_intrcnt_t, const char *); - -#ifdef SMP -void riscv_setup_ipihandler(driver_filter_t *); -void riscv_unmask_ipi(void); -#endif enum { IRQ_SOFTWARE_USER, @@ -75,7 +54,6 @@ enum { IRQ_EXTERNAL_SUPERVISOR, IRQ_EXTERNAL_HYPERVISOR, IRQ_EXTERNAL_MACHINE, - INTC_NIRQS }; #endif /* !_MACHINE_INTR_MACHDEP_H_ */ diff --git a/sys/riscv/include/smp.h b/sys/riscv/include/smp.h index 8bafc1ac6a3..863f0bbd6d2 100644 --- a/sys/riscv/include/smp.h +++ b/sys/riscv/include/smp.h @@ -37,14 +37,15 @@ #include -#define IPI_AST (1 << 0) -#define IPI_PREEMPT (1 << 1) -#define IPI_RENDEZVOUS (1 << 2) -#define IPI_STOP (1 << 3) -#define IPI_STOP_HARD (1 << 4) -#define IPI_HARDCLOCK (1 << 5) - -#define INTR_IPI_COUNT 1 +enum { + IPI_AST, + IPI_PREEMPT, + IPI_RENDEZVOUS, + IPI_STOP, + IPI_STOP_HARD, + IPI_HARDCLOCK, + INTR_IPI_COUNT +}; void ipi_all_but_self(u_int ipi); void ipi_cpu(int cpu, u_int ipi); diff --git a/sys/riscv/riscv/intc.c b/sys/riscv/riscv/intc.c new file mode 100644 index 00000000000..399bb05bbcf --- /dev/null +++ b/sys/riscv/riscv/intc.c @@ -0,0 +1,311 @@ +/*- + * Copyright (c) 2015-2017 Ruslan Bukin + * All rights reserved. + * Copyright (c) 2021 Jessica Clarke + * + * Portions of this software were developed by SRI International and the + * University of Cambridge Computer Laboratory under DARPA/AFRL contract + * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. + * + * Portions of this software were developed by the University of Cambridge + * Computer Laboratory as part of the CTSRD Project, with support from the + * UK Higher Education Innovation Fund (HEIF). + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +#include "pic_if.h" + +#define INTC_NIRQS 16 + +struct intc_irqsrc { + struct intr_irqsrc isrc; + u_int irq; +}; + +struct intc_softc { + device_t dev; + struct intc_irqsrc isrcs[INTC_NIRQS]; +}; + +static int intc_intr(void *arg); + +static phandle_t +intc_ofw_find(device_t dev, uint32_t hartid) +{ + phandle_t node; + pcell_t reg; + + node = OF_finddevice("/cpus"); + if (node == -1) { + device_printf(dev, "Can't find cpus node\n"); + return ((phandle_t)-1); + } + + for (node = OF_child(node); node != 0; node = OF_peer(node)) { + if (!ofw_bus_node_status_okay(node)) + continue; + + if (!ofw_bus_node_is_compatible(node, "riscv")) + continue; + + if (OF_searchencprop(node, "reg", ®, sizeof(reg)) == -1) + continue; + + if (reg == hartid) + break; + } + + if (node == 0) { + device_printf(dev, "Can't find boot cpu node\n"); + return ((phandle_t)-1); + } + + for (node = OF_child(node); node != 0; node = OF_peer(node)) { + if (!ofw_bus_node_status_okay(node)) + continue; + + if (ofw_bus_node_is_compatible(node, "riscv,cpu-intc")) + break; + } + + if (node == 0) { + device_printf(dev, + "Can't find boot cpu local interrupt controller\n"); + return ((phandle_t)-1); + } + + return (node); +} + +static void +intc_identify(driver_t *driver, device_t parent) +{ + device_t dev; + phandle_t node; + + if (device_find_child(parent, "intc", -1) != NULL) + return; + + node = intc_ofw_find(parent, PCPU_GET(hart)); + if (node == -1) + return; + + dev = simplebus_add_device(parent, node, 0, "intc", -1, NULL); + if (dev == NULL) + device_printf(parent, "Can't add intc child\n"); +} + +static int +intc_probe(device_t dev) +{ + device_set_desc(dev, "RISC-V Local Interrupt Controller"); + + return (BUS_PROBE_NOWILDCARD); +} + +static int +intc_attach(device_t dev) +{ + struct intc_irqsrc *isrcs; + struct intc_softc *sc; + struct intr_pic *pic; + const char *name; + phandle_t xref; + u_int flags; + int i, error; + + sc = device_get_softc(dev); + sc->dev = dev; + + name = device_get_nameunit(dev); + xref = OF_xref_from_node(ofw_bus_get_node(dev)); + + isrcs = sc->isrcs; + for (i = 0; i < INTC_NIRQS; i++) { + isrcs[i].irq = i; + flags = i == IRQ_SOFTWARE_SUPERVISOR + ? INTR_ISRCF_IPI : INTR_ISRCF_PPI; + error = intr_isrc_register(&isrcs[i].isrc, sc->dev, flags, + "%s,%u", name, i); + if (error != 0) { + device_printf(dev, "Can't register interrupt %d\n", i); + return (error); + } + } + + pic = intr_pic_register(sc->dev, xref); + if (pic == NULL) + return (ENXIO); + + return (intr_pic_claim_root(sc->dev, xref, intc_intr, sc)); +} + +static void +intc_disable_intr(device_t dev, struct intr_irqsrc *isrc) +{ + u_int irq; + + irq = ((struct intc_irqsrc *)isrc)->irq; + if (irq >= INTC_NIRQS) + panic("%s: Unsupported IRQ %u", __func__, irq); + + csr_clear(sie, 1ul << irq); +} + +static void +intc_enable_intr(device_t dev, struct intr_irqsrc *isrc) +{ + u_int irq; + + irq = ((struct intc_irqsrc *)isrc)->irq; + if (irq >= INTC_NIRQS) + panic("%s: Unsupported IRQ %u", __func__, irq); + + csr_set(sie, 1ul << irq); +} + +static int +intc_map_intr(device_t dev, struct intr_map_data *data, + struct intr_irqsrc **isrcp) +{ + struct intr_map_data_fdt *daf; + struct intc_softc *sc; + + sc = device_get_softc(dev); + + if (data->type != INTR_MAP_DATA_FDT) + return (ENOTSUP); + + daf = (struct intr_map_data_fdt *)data; + if (daf->ncells != 1 || daf->cells[0] >= INTC_NIRQS) + return (EINVAL); + + *isrcp = &sc->isrcs[daf->cells[0]].isrc; + + return (0); +} + +static int +intc_setup_intr(device_t dev, struct intr_irqsrc *isrc, + struct resource *res, struct intr_map_data *data) +{ + if (isrc->isrc_flags & INTR_ISRCF_PPI) + CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu); + + return (0); +} + +#ifdef SMP +static void +intc_init_secondary(device_t dev) +{ + struct intc_softc *sc; + struct intr_irqsrc *isrc; + u_int cpu, irq; + + sc = device_get_softc(dev); + cpu = PCPU_GET(cpuid); + + /* Unmask attached interrupts */ + for (irq = 0; irq < INTC_NIRQS; irq++) { + isrc = &sc->isrcs[irq].isrc; + if (intr_isrc_init_on_cpu(isrc, cpu)) + intc_enable_intr(dev, isrc); + } +} +#endif + +static int +intc_intr(void *arg) +{ + struct trapframe *frame; + struct intc_softc *sc; + uint64_t active_irq; + struct intc_irqsrc *src; + + sc = arg; + frame = curthread->td_intr_frame; + + KASSERT((frame->tf_scause & SCAUSE_INTR) != 0, + ("%s: not an interrupt frame", __func__)); + + active_irq = frame->tf_scause & SCAUSE_CODE; + + if (active_irq >= INTC_NIRQS) + return (FILTER_HANDLED); + + src = &sc->isrcs[active_irq]; + if (intr_isrc_dispatch(&src->isrc, frame) != 0) { + intc_disable_intr(sc->dev, &src->isrc); + device_printf(sc->dev, "Stray irq %lu disabled\n", + active_irq); + } + + return (FILTER_HANDLED); +} + +static device_method_t intc_methods[] = { + /* Device interface */ + DEVMETHOD(device_identify, intc_identify), + DEVMETHOD(device_probe, intc_probe), + DEVMETHOD(device_attach, intc_attach), + + /* Interrupt controller interface */ + DEVMETHOD(pic_disable_intr, intc_disable_intr), + DEVMETHOD(pic_enable_intr, intc_enable_intr), + DEVMETHOD(pic_map_intr, intc_map_intr), + DEVMETHOD(pic_setup_intr, intc_setup_intr), +#ifdef SMP + DEVMETHOD(pic_init_secondary, intc_init_secondary), +#endif + + DEVMETHOD_END +}; + +DEFINE_CLASS_0(intc, intc_driver, intc_methods, sizeof(struct intc_softc)); +EARLY_DRIVER_MODULE(intc, ofwbus, intc_driver, 0, 0, + BUS_PASS_INTERRUPT + BUS_PASS_ORDER_FIRST); diff --git a/sys/riscv/riscv/intr_machdep.c b/sys/riscv/riscv/intr_machdep.c deleted file mode 100644 index 6c00b29c120..00000000000 --- a/sys/riscv/riscv/intr_machdep.c +++ /dev/null @@ -1,274 +0,0 @@ -/*- - * Copyright (c) 2015-2017 Ruslan Bukin - * All rights reserved. - * - * Portions of this software were developed by SRI International and the - * University of Cambridge Computer Laboratory under DARPA/AFRL contract - * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. - * - * Portions of this software were developed by the University of Cambridge - * Computer Laboratory as part of the CTSRD Project, with support from the - * UK Higher Education Innovation Fund (HEIF). - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#ifdef SMP -#include -#endif - -void intr_irq_handler(struct trapframe *tf); - -struct intc_irqsrc { - struct intr_irqsrc isrc; - u_int irq; -}; - -struct intc_irqsrc isrcs[INTC_NIRQS]; - -static void -riscv_mask_irq(void *source) -{ - int irq; - - irq = (int)(uintptr_t)source; - - switch (irq) { - case IRQ_TIMER_SUPERVISOR: - csr_clear(sie, SIE_STIE); - break; - case IRQ_SOFTWARE_USER: - csr_clear(sie, SIE_USIE); - break; - case IRQ_SOFTWARE_SUPERVISOR: - csr_clear(sie, SIE_SSIE); - break; - default: - panic("Unknown irq %d\n", irq); - } -} - -static void -riscv_unmask_irq(void *source) -{ - int irq; - - irq = (int)(uintptr_t)source; - - switch (irq) { - case IRQ_TIMER_SUPERVISOR: - csr_set(sie, SIE_STIE); - break; - case IRQ_SOFTWARE_USER: - csr_set(sie, SIE_USIE); - break; - case IRQ_SOFTWARE_SUPERVISOR: - csr_set(sie, SIE_SSIE); - break; - default: - panic("Unknown irq %d\n", irq); - } -} - -int -riscv_setup_intr(const char *name, driver_filter_t *filt, - void (*handler)(void*), void *arg, int irq, int flags, void **cookiep) -{ - struct intr_irqsrc *isrc; - int error; - - if (irq < 0 || irq >= INTC_NIRQS) - panic("%s: unknown intr %d", __func__, irq); - - isrc = &isrcs[irq].isrc; - if (isrc->isrc_event == NULL) { - error = intr_event_create(&isrc->isrc_event, isrc, 0, irq, - riscv_mask_irq, riscv_unmask_irq, NULL, NULL, "int%d", irq); - if (error) - return (error); - riscv_unmask_irq((void*)(uintptr_t)irq); - } - - error = intr_event_add_handler(isrc->isrc_event, name, - filt, handler, arg, intr_priority(flags), flags, cookiep); - if (error) { - printf("Failed to setup intr: %d\n", irq); - return (error); - } - - return (0); -} - -int -riscv_teardown_intr(void *ih) -{ - - /* TODO */ - - return (0); -} - -void -riscv_cpu_intr(struct trapframe *frame) -{ - struct intr_irqsrc *isrc; - int active_irq; - - KASSERT((frame->tf_scause & SCAUSE_INTR) != 0, - ("riscv_cpu_intr: wrong frame passed")); - - active_irq = frame->tf_scause & SCAUSE_CODE; - - CTR3(KTR_TRAP, "%s: irq=%d, umode=%d", __func__, active_irq, - TRAPF_USERMODE(frame)); - - switch (active_irq) { - case IRQ_SOFTWARE_USER: - case IRQ_SOFTWARE_SUPERVISOR: - case IRQ_TIMER_SUPERVISOR: - critical_enter(); - isrc = &isrcs[active_irq].isrc; - if (intr_isrc_dispatch(isrc, frame) != 0) - printf("stray interrupt %d\n", active_irq); - critical_exit(); - break; - case IRQ_EXTERNAL_SUPERVISOR: - intr_irq_handler(frame); - break; - } -} - -#ifdef SMP -void -riscv_setup_ipihandler(driver_filter_t *filt) -{ - - riscv_setup_intr("ipi", filt, NULL, NULL, IRQ_SOFTWARE_SUPERVISOR, - INTR_TYPE_MISC, NULL); -} - -void -riscv_unmask_ipi(void) -{ - - csr_set(sie, SIE_SSIE); -} - -/* Sending IPI */ -static void -ipi_send(struct pcpu *pc, int ipi) -{ - u_long mask; - - CTR3(KTR_SMP, "%s: cpu: %d, ipi: %x", __func__, pc->pc_cpuid, ipi); - - atomic_set_32(&pc->pc_pending_ipis, ipi); - mask = (1 << pc->pc_hart); - - sbi_send_ipi(&mask); - - CTR1(KTR_SMP, "%s: sent", __func__); -} - -void -ipi_all_but_self(u_int ipi) -{ - cpuset_t other_cpus; - - other_cpus = all_cpus; - CPU_CLR(PCPU_GET(cpuid), &other_cpus); - - CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); - ipi_selected(other_cpus, ipi); -} - -void -ipi_cpu(int cpu, u_int ipi) -{ - cpuset_t cpus; - - CPU_ZERO(&cpus); - CPU_SET(cpu, &cpus); - - ipi_send(cpuid_to_pcpu[cpu], ipi); -} - -void -ipi_selected(cpuset_t cpus, u_int ipi) -{ - struct pcpu *pc; - u_long mask; - - CTR1(KTR_SMP, "ipi_selected: ipi: %x", ipi); - - mask = 0; - STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { - if (CPU_ISSET(pc->pc_cpuid, &cpus)) { - CTR3(KTR_SMP, "%s: pc: %p, ipi: %x\n", __func__, pc, - ipi); - atomic_set_32(&pc->pc_pending_ipis, ipi); - mask |= (1 << pc->pc_hart); - } - } - sbi_send_ipi(&mask); -} -#endif - -/* Interrupt machdep initialization routine. */ -static void -intc_init(void *dummy __unused) -{ - int error; - int i; - - for (i = 0; i < INTC_NIRQS; i++) { - isrcs[i].irq = i; - error = intr_isrc_register(&isrcs[i].isrc, NULL, - 0, "intc,%u", i); - if (error != 0) - printf("Can't register interrupt %d\n", i); - } -} - -SYSINIT(intc_init, SI_SUB_INTR, SI_ORDER_MIDDLE, intc_init, NULL); diff --git a/sys/riscv/riscv/mp_machdep.c b/sys/riscv/riscv/mp_machdep.c index 2c144e457b4..be5c50485bf 100644 --- a/sys/riscv/riscv/mp_machdep.c +++ b/sys/riscv/riscv/mp_machdep.c @@ -83,7 +83,11 @@ static device_identify_t riscv64_cpu_identify; static device_probe_t riscv64_cpu_probe; static device_attach_t riscv64_cpu_attach; -static int ipi_handler(void *); +static void ipi_ast(void *); +static void ipi_hardclock(void *); +static void ipi_preempt(void *); +static void ipi_rendezvous(void *); +static void ipi_stop(void *); extern uint32_t boot_hart; extern cpuset_t all_harts; @@ -191,8 +195,13 @@ release_aps(void *dummy __unused) if (mp_ncpus == 1) return; - /* Setup the IPI handler */ - riscv_setup_ipihandler(ipi_handler); + /* Setup the IPI handlers */ + intr_ipi_setup(IPI_AST, "ast", ipi_ast, NULL); + intr_ipi_setup(IPI_PREEMPT, "preempt", ipi_preempt, NULL); + intr_ipi_setup(IPI_RENDEZVOUS, "rendezvous", ipi_rendezvous, NULL); + intr_ipi_setup(IPI_STOP, "stop", ipi_stop, NULL); + intr_ipi_setup(IPI_STOP_HARD, "stop hard", ipi_stop, NULL); + intr_ipi_setup(IPI_HARDCLOCK, "hardclock", ipi_hardclock, NULL); atomic_store_rel_int(&aps_ready, 1); @@ -244,17 +253,14 @@ init_secondary(uint64_t hart) pcpup->pc_curthread = pcpup->pc_idlethread; schedinit_ap(); - /* Enable software interrupts */ - riscv_unmask_ipi(); + /* Setup and enable interrupts */ + intr_pic_init_secondary(); #ifndef EARLY_AP_STARTUP /* Start per-CPU event timers. */ cpu_initclocks_ap(); #endif - /* Enable external (PLIC) interrupts */ - csr_set(sie, SIE_SEIE); - /* Activate this hart in the kernel pmap. */ CPU_SET_ATOMIC(hart, &kernel_pmap->pm_active); @@ -308,74 +314,59 @@ smp_after_idle_runnable(void *arg __unused) SYSINIT(smp_after_idle_runnable, SI_SUB_SMP, SI_ORDER_ANY, smp_after_idle_runnable, NULL); -static int -ipi_handler(void *arg) +static void +ipi_ast(void *dummy __unused) { - u_int ipi_bitmap; - u_int cpu, ipi; - int bit; + CTR0(KTR_SMP, "IPI_AST"); +} - csr_clear(sip, SIP_SSIP); +static void +ipi_preempt(void *dummy __unused) +{ + CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__); + sched_preempt(curthread); +} + +static void +ipi_rendezvous(void *dummy __unused) +{ + CTR0(KTR_SMP, "IPI_RENDEZVOUS"); + smp_rendezvous_action(); +} + +static void +ipi_stop(void *dummy __unused) +{ + u_int cpu; + + CTR0(KTR_SMP, "IPI_STOP"); cpu = PCPU_GET(cpuid); + savectx(&stoppcbs[cpu]); - mb(); + /* Indicate we are stopped */ + CPU_SET_ATOMIC(cpu, &stopped_cpus); - ipi_bitmap = atomic_readandclear_int(PCPU_PTR(pending_ipis)); - if (ipi_bitmap == 0) - return (FILTER_HANDLED); + /* Wait for restart */ + while (!CPU_ISSET(cpu, &started_cpus)) + cpu_spinwait(); - while ((bit = ffs(ipi_bitmap))) { - bit = (bit - 1); - ipi = (1 << bit); - ipi_bitmap &= ~ipi; + CPU_CLR_ATOMIC(cpu, &started_cpus); + CPU_CLR_ATOMIC(cpu, &stopped_cpus); + CTR0(KTR_SMP, "IPI_STOP (restart)"); - mb(); + /* + * The kernel debugger might have set a breakpoint, + * so flush the instruction cache. + */ + fence_i(); +} - switch (ipi) { - case IPI_AST: - CTR0(KTR_SMP, "IPI_AST"); - break; - case IPI_PREEMPT: - CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__); - sched_preempt(curthread); - break; - case IPI_RENDEZVOUS: - CTR0(KTR_SMP, "IPI_RENDEZVOUS"); - smp_rendezvous_action(); - break; - case IPI_STOP: - case IPI_STOP_HARD: - CTR0(KTR_SMP, (ipi == IPI_STOP) ? "IPI_STOP" : "IPI_STOP_HARD"); - savectx(&stoppcbs[cpu]); - - /* Indicate we are stopped */ - CPU_SET_ATOMIC(cpu, &stopped_cpus); - - /* Wait for restart */ - while (!CPU_ISSET(cpu, &started_cpus)) - cpu_spinwait(); - - CPU_CLR_ATOMIC(cpu, &started_cpus); - CPU_CLR_ATOMIC(cpu, &stopped_cpus); - CTR0(KTR_SMP, "IPI_STOP (restart)"); - - /* - * The kernel debugger might have set a breakpoint, - * so flush the instruction cache. - */ - fence_i(); - break; - case IPI_HARDCLOCK: - CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__); - hardclockintr(); - break; - default: - panic("Unknown IPI %#0x on cpu %d", ipi, curcpu); - } - } - - return (FILTER_HANDLED); +static void +ipi_hardclock(void *dummy __unused) +{ + CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__); + hardclockintr(); } struct cpu_group * @@ -574,3 +565,34 @@ cpu_mp_setmaxid(void) } } } + +void +ipi_all_but_self(u_int ipi) +{ + cpuset_t other_cpus; + + other_cpus = all_cpus; + CPU_CLR(PCPU_GET(cpuid), &other_cpus); + + CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); + intr_ipi_send(other_cpus, ipi); +} + +void +ipi_cpu(int cpu, u_int ipi) +{ + cpuset_t cpus; + + CPU_ZERO(&cpus); + CPU_SET(cpu, &cpus); + + CTR3(KTR_SMP, "%s: cpu: %d, ipi: %x", __func__, cpu, ipi); + intr_ipi_send(cpus, ipi); +} + +void +ipi_selected(cpuset_t cpus, u_int ipi) +{ + CTR1(KTR_SMP, "ipi_selected: ipi: %x", ipi); + intr_ipi_send(cpus, ipi); +} diff --git a/sys/riscv/riscv/plic.c b/sys/riscv/riscv/plic.c index 1e8a0312703..406aff5bbff 100644 --- a/sys/riscv/riscv/plic.c +++ b/sys/riscv/riscv/plic.c @@ -91,16 +91,18 @@ struct plic_context { struct plic_softc { device_t dev; - struct resource * intc_res; + struct resource *mem_res; + struct resource *irq_res; + void *ih; struct plic_irqsrc isrcs[PLIC_MAX_IRQS]; struct plic_context contexts[MAXCPU]; int ndev; }; #define RD4(sc, reg) \ - bus_read_4(sc->intc_res, (reg)) + bus_read_4(sc->mem_res, (reg)) #define WR4(sc, reg, val) \ - bus_write_4(sc->intc_res, (reg), (val)) + bus_write_4(sc->mem_res, (reg), (val)) static u_int plic_irq_cpu; @@ -276,9 +278,9 @@ plic_attach(device_t dev) /* Request memory resources */ rid = 0; - sc->intc_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, + sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); - if (sc->intc_res == NULL) { + if (sc->mem_res == NULL) { device_printf(dev, "Error: could not allocate memory resources\n"); return (ENXIO); @@ -302,7 +304,7 @@ plic_attach(device_t dev) * * This is tricky for a few reasons. The PLIC divides the interrupt * enable, threshold, and claim bits by "context", where each context - * routes to a Core-Local Interrupt Controller (CLIC). + * routes to a core's local interrupt controller. * * The tricky part is that the PLIC spec imposes no restrictions on how * these contexts are laid out. So for example, there is no guarantee @@ -316,10 +318,15 @@ plic_attach(device_t dev) * entries that are not for supervisor external interrupts. * * 2. Walk up the device tree to find the corresponding CPU, and grab - * it's hart ID. + * its hart ID. * * 3. Convert the hart to a cpuid, and calculate the register offsets * based on the context number. + * + * 4. Save the index for the boot hart's S-mode external interrupt in + * order to allocate and setup the corresponding resource, since the + * local interrupt controller newbus device is associated with that + * specific node. */ nintr = OF_getencprop_alloc_multi(node, "interrupts-extended", sizeof(uint32_t), (void **)&cells); @@ -329,12 +336,16 @@ plic_attach(device_t dev) } /* interrupts-extended is a list of phandles and interrupt types. */ + rid = -1; for (i = 0, context = 0; i < nintr; i += 2, context++) { /* Skip M-mode external interrupts */ if (cells[i + 1] != IRQ_EXTERNAL_SUPERVISOR) continue; - /* Get the hart ID from the CLIC's phandle. */ + /* + * Get the hart ID from the core's interrupt controller + * phandle. + */ hart = plic_get_hartid(dev, OF_node_from_xref(cells[i])); if (hart < 0) { OF_prop_free(cells); @@ -349,6 +360,9 @@ plic_attach(device_t dev) return (ENXIO); } + if (cpu == 0) + rid = i / 2; + /* Set the enable and context register offsets for the CPU. */ sc->contexts[cpu].enable_offset = PLIC_ENABLE_BASE + context * PLIC_ENABLE_STRIDE; @@ -357,6 +371,20 @@ plic_attach(device_t dev) } OF_prop_free(cells); + if (rid == -1) { + device_printf(dev, + "Could not find local interrupt controller\n"); + return (ENXIO); + } + + sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, + RF_ACTIVE); + if (sc->irq_res == NULL) { + device_printf(dev, + "Error: could not allocate IRQ resources\n"); + return (ENXIO); + } + /* Set the threshold for each CPU to accept all priorities. */ CPU_FOREACH(cpu) WR4(sc, PLIC_THRESHOLD(sc, cpu), 0); @@ -366,9 +394,8 @@ plic_attach(device_t dev) if (pic == NULL) return (ENXIO); - csr_set(sie, SIE_SEIE); - - return (intr_pic_claim_root(sc->dev, xref, plic_intr, sc)); + return (bus_setup_intr(dev, sc->irq_res, INTR_TYPE_CLK | INTR_MPSAFE, + plic_intr, NULL, sc, &sc->ih)); } static void diff --git a/sys/riscv/riscv/riscv_console.c b/sys/riscv/riscv/riscv_console.c index de61fe24dce..d6c4ce4540c 100644 --- a/sys/riscv/riscv/riscv_console.c +++ b/sys/riscv/riscv/riscv_console.c @@ -243,8 +243,6 @@ rcons_attach(device_t dev) sc = device_get_softc(dev); sc->dev = dev; - csr_set(sie, SIE_SSIE); - bus_generic_attach(sc->dev); return (0); diff --git a/sys/riscv/riscv/sbi.c b/sys/riscv/riscv/sbi.c index 7c6d1906445..53eadbe54bc 100644 --- a/sys/riscv/riscv/sbi.c +++ b/sys/riscv/riscv/sbi.c @@ -31,6 +31,7 @@ #include #include #include +#include #include #include @@ -45,6 +46,10 @@ struct sbi_softc { device_t dev; }; +struct sbi_devinfo { + struct resource_list rl; +}; + static struct sbi_softc *sbi_softc = NULL; static u_long sbi_spec_version; @@ -347,6 +352,10 @@ static int sbi_attach(device_t dev) { struct sbi_softc *sc; +#ifdef SMP + device_t child; + struct sbi_devinfo *di; +#endif if (sbi_softc != NULL) return (ENXIO); @@ -358,15 +367,49 @@ sbi_attach(device_t dev) EVENTHANDLER_REGISTER(shutdown_final, sbi_shutdown_final, NULL, SHUTDOWN_PRI_LAST); +#ifdef SMP + di = malloc(sizeof(*di), M_DEVBUF, M_WAITOK | M_ZERO); + resource_list_init(&di->rl); + child = device_add_child(dev, "sbi_ipi", -1); + if (child == NULL) { + device_printf(dev, "Could not add sbi_ipi child\n"); + return (ENXIO); + } + + device_set_ivars(child, di); +#endif + return (0); } +static struct resource_list * +sbi_get_resource_list(device_t bus, device_t child) +{ + struct sbi_devinfo *di; + + di = device_get_ivars(child); + KASSERT(di != NULL, ("%s: No devinfo", __func__)); + + return (&di->rl); +} + static device_method_t sbi_methods[] = { /* Device interface */ DEVMETHOD(device_identify, sbi_identify), DEVMETHOD(device_probe, sbi_probe), DEVMETHOD(device_attach, sbi_attach), + /* Bus interface */ + DEVMETHOD(bus_alloc_resource, bus_generic_rl_alloc_resource), + DEVMETHOD(bus_activate_resource, bus_generic_activate_resource), + DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource), + DEVMETHOD(bus_release_resource, bus_generic_rl_release_resource), + DEVMETHOD(bus_setup_intr, bus_generic_setup_intr), + DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr), + DEVMETHOD(bus_get_resource_list, sbi_get_resource_list), + DEVMETHOD(bus_set_resource, bus_generic_rl_set_resource), + DEVMETHOD(bus_get_resource, bus_generic_rl_get_resource), + DEVMETHOD_END }; diff --git a/sys/riscv/riscv/sbi_ipi.c b/sys/riscv/riscv/sbi_ipi.c new file mode 100644 index 00000000000..d694b476f5f --- /dev/null +++ b/sys/riscv/riscv/sbi_ipi.c @@ -0,0 +1,208 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2021 Jessica Clarke + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include + +#include "pic_if.h" + +struct sbi_ipi_softc { + device_t dev; + struct resource *irq_res; + void *ih; + struct intr_irqsrc isrc; + uint32_t pending_ipis[MAXCPU]; +}; + +static void +sbi_ipi_pic_ipi_send(device_t dev, struct intr_irqsrc *isrc, cpuset_t cpus, + u_int ipi) +{ + struct sbi_ipi_softc *sc; + struct pcpu *pc; + u_long mask; + u_int cpu; + + sc = device_get_softc(dev); + + KASSERT(isrc == &sc->isrc, ("%s: not the IPI isrc", __func__)); + KASSERT(ipi < INTR_IPI_COUNT, + ("%s: not a valid IPI: %u", __func__, ipi)); + + mask = 0; + STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { + cpu = pc->pc_cpuid; + if (CPU_ISSET(cpu, &cpus)) { + atomic_set_32(&sc->pending_ipis[cpu], 1u << ipi); + mask |= (1ul << pc->pc_hart); + } + } + sbi_send_ipi(&mask); +} + +static int +sbi_ipi_pic_ipi_setup(device_t dev, u_int ipi, struct intr_irqsrc **isrcp) +{ + struct sbi_ipi_softc *sc; + + sc = device_get_softc(dev); + + KASSERT(ipi < INTR_IPI_COUNT, + ("%s: not a valid IPI: %u", __func__, ipi)); + + *isrcp = &sc->isrc; + + return (0); + +} + +static int +sbi_ipi_intr(void *arg) +{ + struct sbi_ipi_softc *sc; + uint32_t ipi_bitmap; + u_int cpu, ipi; + int bit; + + sc = arg; + + csr_clear(sip, SIP_SSIP); + + cpu = PCPU_GET(cpuid); + + mb(); + + ipi_bitmap = atomic_readandclear_32(&sc->pending_ipis[cpu]); + if (ipi_bitmap == 0) + return (FILTER_HANDLED); + + mb(); + + while ((bit = ffs(ipi_bitmap))) { + ipi = (bit - 1); + ipi_bitmap &= ~(1u << ipi); + + intr_ipi_dispatch(ipi); + } + + return (FILTER_HANDLED); +} + +static int +sbi_ipi_probe(device_t dev) +{ + device_set_desc(dev, "RISC-V SBI Inter-Processor Interrupts"); + + return (BUS_PROBE_NOWILDCARD); +} + +static int +sbi_ipi_attach(device_t dev) +{ + struct sbi_ipi_softc *sc; + const char *name; + int irq, rid, error; + phandle_t iparent; + pcell_t cell; + + sc = device_get_softc(dev); + sc->dev = dev; + + memset(sc->pending_ipis, 0, sizeof(sc->pending_ipis)); + + name = device_get_nameunit(dev); + error = intr_isrc_register(&sc->isrc, sc->dev, INTR_ISRCF_IPI, + "%s,ipi", name); + if (error != 0) { + device_printf(dev, "Can't register interrupt: %d\n", error); + return (ENXIO); + } + + iparent = OF_xref_from_node(ofw_bus_get_node(intr_irq_root_dev)); + cell = IRQ_SOFTWARE_SUPERVISOR; + irq = ofw_bus_map_intr(dev, iparent, 1, &cell); + error = bus_set_resource(dev, SYS_RES_IRQ, 0, irq, 1); + if (error != 0) { + device_printf(dev, "Unable to register IRQ resource\n"); + return (ENXIO); + } + + rid = 0; + sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, + RF_ACTIVE | RF_SHAREABLE); + if (sc->irq_res == NULL) { + device_printf(dev, "Unable to alloc IRQ resource\n"); + return (ENXIO); + } + + error = bus_setup_intr(dev, sc->irq_res, INTR_TYPE_CLK, + sbi_ipi_intr, NULL, sc, &sc->ih); + if (error != 0) { + device_printf(dev, "Unable to setup IRQ resource\n"); + return (ENXIO); + } + + /* TODO: Define a set of priorities once other IPI sources exist */ + error = intr_ipi_pic_register(dev, 0); + if (error != 0) { + device_printf(dev, "Can't register as IPI source: %d\n", error); + return (ENXIO); + } + + return (0); +} + +static device_method_t sbi_ipi_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, sbi_ipi_probe), + DEVMETHOD(device_attach, sbi_ipi_attach), + + /* Interrupt controller interface */ + DEVMETHOD(pic_ipi_send, sbi_ipi_pic_ipi_send), + DEVMETHOD(pic_ipi_setup, sbi_ipi_pic_ipi_setup), + + DEVMETHOD_END +}; + +DEFINE_CLASS_0(sbi_ipi, sbi_ipi_driver, sbi_ipi_methods, + sizeof(struct sbi_ipi_softc)); +/* Local interrupt controller attaches during BUS_PASS_ORDER_FIRST */ +EARLY_DRIVER_MODULE(sbi_ipi, sbi, sbi_ipi_driver, 0, 0, + BUS_PASS_INTERRUPT + BUS_PASS_ORDER_EARLY); diff --git a/sys/riscv/riscv/timer.c b/sys/riscv/riscv/timer.c index bb98aee3e2c..06806872d9f 100644 --- a/sys/riscv/riscv/timer.c +++ b/sys/riscv/riscv/timer.c @@ -43,6 +43,7 @@ #include #include #include +#include #include #include #include @@ -53,9 +54,11 @@ #include #include +#include #include struct riscv_timer_softc { + struct resource *irq_res; void *ih; uint32_t clkfreq; struct eventtimer et; @@ -116,7 +119,6 @@ riscv_timer_et_start(struct eventtimer *et, sbintime_t first, sbintime_t period) if (first != 0) { counts = ((uint32_t)et->et_frequency * first) >> 32; set_timecmp(get_timecount() + counts); - csr_set(sie, SIE_STIE); return (0); } @@ -188,7 +190,9 @@ static int riscv_timer_attach(device_t dev) { struct riscv_timer_softc *sc; - int error; + int irq, rid, error; + phandle_t iparent; + pcell_t cell; sc = device_get_softc(dev); if (riscv_timer_sc != NULL) @@ -204,11 +208,28 @@ riscv_timer_attach(device_t dev) riscv_timer_sc = sc; + iparent = OF_xref_from_node(ofw_bus_get_node(intr_irq_root_dev)); + cell = IRQ_TIMER_SUPERVISOR; + irq = ofw_bus_map_intr(dev, iparent, 1, &cell); + error = bus_set_resource(dev, SYS_RES_IRQ, 0, irq, 1); + if (error != 0) { + device_printf(dev, "Unable to register IRQ resource\n"); + return (ENXIO); + } + + rid = 0; + sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, + RF_ACTIVE); + if (sc->irq_res == NULL) { + device_printf(dev, "Unable to alloc IRQ resource\n"); + return (ENXIO); + } + /* Setup IRQs handler */ - error = riscv_setup_intr(device_get_nameunit(dev), riscv_timer_intr, - NULL, sc, IRQ_TIMER_SUPERVISOR, INTR_TYPE_CLK, &sc->ih); - if (error) { - device_printf(dev, "Unable to alloc int resource.\n"); + error = bus_setup_intr(dev, sc->irq_res, INTR_TYPE_CLK, + riscv_timer_intr, NULL, sc, &sc->ih); + if (error != 0) { + device_printf(dev, "Unable to setup IRQ resource\n"); return (ENXIO); } diff --git a/sys/riscv/riscv/trap.c b/sys/riscv/riscv/trap.c index f54efc32494..3a7b3fba26b 100644 --- a/sys/riscv/riscv/trap.c +++ b/sys/riscv/riscv/trap.c @@ -74,6 +74,8 @@ #include #endif +void intr_irq_handler(struct trapframe *tf); + int (*dtrace_invop_jump_addr)(struct trapframe *); /* Called from exception.S */ @@ -317,7 +319,7 @@ do_trap_supervisor(struct trapframe *frame) exception = frame->tf_scause & SCAUSE_CODE; if ((frame->tf_scause & SCAUSE_INTR) != 0) { /* Interrupt */ - riscv_cpu_intr(frame); + intr_irq_handler(frame); return; } @@ -398,7 +400,7 @@ do_trap_user(struct trapframe *frame) exception = frame->tf_scause & SCAUSE_CODE; if ((frame->tf_scause & SCAUSE_INTR) != 0) { /* Interrupt */ - riscv_cpu_intr(frame); + intr_irq_handler(frame); return; } intr_enable();