From e3bdf3da769a55f0944d9c337bb4d91b6435f02c Mon Sep 17 00:00:00 2001 From: Alexander Motin Date: Tue, 31 Aug 2021 13:34:48 -0400 Subject: [PATCH] nvme(4): Add MSI and single MSI-X support. If we can't allocate more MSI-X vectors, accept using single shared. If we can't allocate any MSI-X, try to allocate 2 MSI vectors, but accept single shared. If still no luck, fall back to shared INTx. This provides maximal flexibility in some limited scenarios. For example, vmd(4) does not support INTx and can handle only limited number of MSI/MSI-X vectors without sharing. MFC after: 1 week --- sys/dev/nvme/nvme_ahci.c | 9 ++-- sys/dev/nvme/nvme_ctrlr.c | 2 +- sys/dev/nvme/nvme_pci.c | 92 +++++++++++++++++++++++-------------- sys/dev/nvme/nvme_private.h | 4 +- sys/dev/nvme/nvme_qpair.c | 14 ++++-- 5 files changed, 73 insertions(+), 48 deletions(-) diff --git a/sys/dev/nvme/nvme_ahci.c b/sys/dev/nvme/nvme_ahci.c index 1037fab6666..8542f6f5524 100644 --- a/sys/dev/nvme/nvme_ahci.c +++ b/sys/dev/nvme/nvme_ahci.c @@ -87,19 +87,18 @@ nvme_ahci_attach(device_t dev) ctrlr->rid = 0; ctrlr->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &ctrlr->rid, RF_SHAREABLE | RF_ACTIVE); - if (ctrlr->res == NULL) { - nvme_printf(ctrlr, "unable to allocate shared IRQ\n"); + nvme_printf(ctrlr, "unable to allocate shared interrupt\n"); ret = ENOMEM; goto bad; } - ctrlr->msix_enabled = 0; + ctrlr->msi_count = 0; ctrlr->num_io_queues = 1; if (bus_setup_intr(dev, ctrlr->res, - INTR_TYPE_MISC | INTR_MPSAFE, NULL, nvme_ctrlr_intx_handler, + INTR_TYPE_MISC | INTR_MPSAFE, NULL, nvme_ctrlr_shared_handler, ctrlr, &ctrlr->tag) != 0) { - nvme_printf(ctrlr, "unable to setup intx handler\n"); + nvme_printf(ctrlr, "unable to setup shared interrupt\n"); ret = ENOMEM; goto bad; } diff --git a/sys/dev/nvme/nvme_ctrlr.c b/sys/dev/nvme/nvme_ctrlr.c index c4a41c453b6..52eab6254d8 100644 --- a/sys/dev/nvme/nvme_ctrlr.c +++ b/sys/dev/nvme/nvme_ctrlr.c @@ -1203,7 +1203,7 @@ nvme_ctrlr_poll(struct nvme_controller *ctrlr) * interrupts in the controller. */ void -nvme_ctrlr_intx_handler(void *arg) +nvme_ctrlr_shared_handler(void *arg) { struct nvme_controller *ctrlr = arg; diff --git a/sys/dev/nvme/nvme_pci.c b/sys/dev/nvme/nvme_pci.c index 1b28ab4d40e..e87860bde7e 100644 --- a/sys/dev/nvme/nvme_pci.c +++ b/sys/dev/nvme/nvme_pci.c @@ -47,7 +47,7 @@ static int nvme_pci_detach(device_t); static int nvme_pci_suspend(device_t); static int nvme_pci_resume(device_t); -static void nvme_ctrlr_setup_interrupts(struct nvme_controller *ctrlr); +static int nvme_ctrlr_setup_interrupts(struct nvme_controller *ctrlr); static device_method_t nvme_pci_methods[] = { /* Device interface */ @@ -188,7 +188,9 @@ nvme_pci_attach(device_t dev) if (status != 0) goto bad; pci_enable_busmaster(dev); - nvme_ctrlr_setup_interrupts(ctrlr); + status = nvme_ctrlr_setup_interrupts(ctrlr); + if (status != 0) + goto bad; return nvme_attach(dev); bad: if (ctrlr->resource != NULL) { @@ -208,7 +210,7 @@ bad: bus_release_resource(dev, SYS_RES_IRQ, rman_get_rid(ctrlr->res), ctrlr->res); - if (ctrlr->msix_enabled) + if (ctrlr->msi_count > 0) pci_release_msi(dev); return status; @@ -221,54 +223,60 @@ nvme_pci_detach(device_t dev) int rv; rv = nvme_detach(dev); - if (ctrlr->msix_enabled) + if (ctrlr->msi_count > 0) pci_release_msi(dev); pci_disable_busmaster(dev); return (rv); } static int -nvme_ctrlr_configure_intx(struct nvme_controller *ctrlr) +nvme_ctrlr_setup_shared(struct nvme_controller *ctrlr, int rid) { + int error; - ctrlr->msix_enabled = 0; ctrlr->num_io_queues = 1; - ctrlr->rid = 0; + ctrlr->rid = rid; ctrlr->res = bus_alloc_resource_any(ctrlr->dev, SYS_RES_IRQ, &ctrlr->rid, RF_SHAREABLE | RF_ACTIVE); - if (ctrlr->res == NULL) { - nvme_printf(ctrlr, "unable to allocate shared IRQ\n"); + nvme_printf(ctrlr, "unable to allocate shared interrupt\n"); return (ENOMEM); } - if (bus_setup_intr(ctrlr->dev, ctrlr->res, - INTR_TYPE_MISC | INTR_MPSAFE, NULL, nvme_ctrlr_intx_handler, - ctrlr, &ctrlr->tag) != 0) { - nvme_printf(ctrlr, "unable to setup intx handler\n"); - return (ENOMEM); + error = bus_setup_intr(ctrlr->dev, ctrlr->res, + INTR_TYPE_MISC | INTR_MPSAFE, NULL, nvme_ctrlr_shared_handler, + ctrlr, &ctrlr->tag); + if (error) { + nvme_printf(ctrlr, "unable to setup shared interrupt\n"); + return (error); } return (0); } -static void +static int nvme_ctrlr_setup_interrupts(struct nvme_controller *ctrlr) { device_t dev; int force_intx, num_io_queues, per_cpu_io_queues; int min_cpus_per_ioq; - int num_vectors_requested, num_vectors_allocated; + int num_vectors_requested; dev = ctrlr->dev; force_intx = 0; TUNABLE_INT_FETCH("hw.nvme.force_intx", &force_intx); - if (force_intx || pci_msix_count(dev) < 2) { - nvme_ctrlr_configure_intx(ctrlr); - return; - } + if (force_intx) + return (nvme_ctrlr_setup_shared(ctrlr, 0)); + if (pci_msix_count(dev) == 0) + goto msi; + + /* + * Try to allocate one MSI-X per core for I/O queues, plus one + * for admin queue, but accept single shared MSI-X if have to. + * Fall back to MSI if can't get any MSI-X. + */ num_io_queues = mp_ncpus; TUNABLE_INT_FETCH("hw.nvme.num_io_queues", &num_io_queues); if (num_io_queues < 1 || num_io_queues > mp_ncpus) @@ -286,31 +294,45 @@ nvme_ctrlr_setup_interrupts(struct nvme_controller *ctrlr) max(1, mp_ncpus / min_cpus_per_ioq)); } - num_io_queues = min(num_io_queues, pci_msix_count(dev) - 1); + num_io_queues = min(num_io_queues, max(1, pci_msix_count(dev) - 1)); again: if (num_io_queues > vm_ndomains) num_io_queues -= num_io_queues % vm_ndomains; - /* One vector for per core I/O queue, plus one vector for admin queue. */ - num_vectors_requested = num_io_queues + 1; - num_vectors_allocated = num_vectors_requested; - if (pci_alloc_msix(dev, &num_vectors_allocated) != 0) { - nvme_ctrlr_configure_intx(ctrlr); - return; + num_vectors_requested = min(num_io_queues + 1, pci_msix_count(dev)); + ctrlr->msi_count = num_vectors_requested; + if (pci_alloc_msix(dev, &ctrlr->msi_count) != 0) { + nvme_printf(ctrlr, "unable to allocate MSI-X\n"); + ctrlr->msi_count = 0; + goto msi; } - if (num_vectors_allocated < 2) { + if (ctrlr->msi_count == 1) + return (nvme_ctrlr_setup_shared(ctrlr, 1)); + if (ctrlr->msi_count != num_vectors_requested) { pci_release_msi(dev); - nvme_ctrlr_configure_intx(ctrlr); - return; - } - if (num_vectors_allocated != num_vectors_requested) { - pci_release_msi(dev); - num_io_queues = num_vectors_allocated - 1; + num_io_queues = ctrlr->msi_count - 1; goto again; } - ctrlr->msix_enabled = 1; ctrlr->num_io_queues = num_io_queues; + return (0); + +msi: + /* + * Try to allocate 2 MSIs (admin and I/O queues), but accept single + * shared if have to. Fall back to INTx if can't get any MSI. + */ + ctrlr->msi_count = min(pci_msi_count(dev), 2); + if (ctrlr->msi_count > 0) { + if (pci_alloc_msi(dev, &ctrlr->msi_count) != 0) { + nvme_printf(ctrlr, "unable to allocate MSI\n"); + ctrlr->msi_count = 0; + } else if (ctrlr->msi_count == 2) { + ctrlr->num_io_queues = 1; + return (0); + } + } + return (nvme_ctrlr_setup_shared(ctrlr, ctrlr->msi_count > 0 ? 1 : 0)); } static int diff --git a/sys/dev/nvme/nvme_private.h b/sys/dev/nvme/nvme_private.h index ee0a909e24e..fba1b406e9c 100644 --- a/sys/dev/nvme/nvme_private.h +++ b/sys/dev/nvme/nvme_private.h @@ -244,7 +244,7 @@ struct nvme_controller { int bar4_resource_id; struct resource *bar4_resource; - uint32_t msix_enabled; + int msi_count; uint32_t enable_aborts; uint32_t num_io_queues; @@ -553,7 +553,7 @@ void nvme_notify_fail_consumers(struct nvme_controller *ctrlr); void nvme_notify_new_controller(struct nvme_controller *ctrlr); void nvme_notify_ns(struct nvme_controller *ctrlr, int nsid); -void nvme_ctrlr_intx_handler(void *arg); +void nvme_ctrlr_shared_handler(void *arg); void nvme_ctrlr_poll(struct nvme_controller *ctrlr); int nvme_ctrlr_suspend(struct nvme_controller *ctrlr); diff --git a/sys/dev/nvme/nvme_qpair.c b/sys/dev/nvme/nvme_qpair.c index 4402d1000e6..eea87e299d3 100644 --- a/sys/dev/nvme/nvme_qpair.c +++ b/sys/dev/nvme/nvme_qpair.c @@ -652,7 +652,7 @@ nvme_qpair_process_completions(struct nvme_qpair *qpair) } static void -nvme_qpair_msix_handler(void *arg) +nvme_qpair_msi_handler(void *arg) { struct nvme_qpair *qpair = arg; @@ -670,7 +670,7 @@ nvme_qpair_construct(struct nvme_qpair *qpair, uint8_t *queuemem, *prpmem, *prp_list; int i, err; - qpair->vector = ctrlr->msix_enabled ? qpair->id : 0; + qpair->vector = ctrlr->msi_count > 1 ? qpair->id : 0; qpair->num_entries = num_entries; qpair->num_trackers = num_trackers; qpair->ctrlr = ctrlr; @@ -795,7 +795,7 @@ nvme_qpair_construct(struct nvme_qpair *qpair, qpair->num_entries, M_NVME, DOMAINSET_PREF(qpair->domain), M_ZERO | M_WAITOK); - if (ctrlr->msix_enabled) { + if (ctrlr->msi_count > 1) { /* * MSI-X vector resource IDs start at 1, so we add one to * the queue's vector to get the corresponding rid to use. @@ -804,10 +804,14 @@ nvme_qpair_construct(struct nvme_qpair *qpair, qpair->res = bus_alloc_resource_any(ctrlr->dev, SYS_RES_IRQ, &qpair->rid, RF_ACTIVE); + if (qpair->res == NULL) { + nvme_printf(ctrlr, "unable to allocate MSI\n"); + goto out; + } if (bus_setup_intr(ctrlr->dev, qpair->res, INTR_TYPE_MISC | INTR_MPSAFE, NULL, - nvme_qpair_msix_handler, qpair, &qpair->tag) != 0) { - nvme_printf(ctrlr, "unable to setup intx handler\n"); + nvme_qpair_msi_handler, qpair, &qpair->tag) != 0) { + nvme_printf(ctrlr, "unable to setup MSI\n"); goto out; } if (qpair->id == 0) {