From 7ab9e36fc54445fd72741fb0c85d84694da046f4 Mon Sep 17 00:00:00 2001 From: Alexander Motin Date: Mon, 6 Nov 2023 11:05:48 -0500 Subject: [PATCH] nvme: Introduce longer timeouts for admin queue KIOXIA CD8 SSDs routinely take ~25 seconds to delete non-empty namespace. In some cases like hot-plug it takes longer, triggering timeout and controller resets after just 30 seconds. Linux for many years has separate 60 seconds timeout for admin queue. This patch does the same. And it is good to be consistent. Sponsored by: iXsystems, Inc. Reviewed by: imp MFC after: 1 week Differential Revision: https://reviews.freebsd.org/D42454 (cherry picked from commit 8d6c0743e36e3cff9279c40468711a82db98df23) --- sys/dev/nvme/nvme_ctrlr.c | 6 ++++++ sys/dev/nvme/nvme_private.h | 2 ++ sys/dev/nvme/nvme_qpair.c | 2 ++ sys/dev/nvme/nvme_sysctl.c | 15 ++++++++++----- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/sys/dev/nvme/nvme_ctrlr.c b/sys/dev/nvme/nvme_ctrlr.c index ef4d7daa6ef..3eca24f1f6a 100644 --- a/sys/dev/nvme/nvme_ctrlr.c +++ b/sys/dev/nvme/nvme_ctrlr.c @@ -1451,6 +1451,12 @@ nvme_ctrlr_construct(struct nvme_controller *ctrlr, device_t dev) to = NVME_CAP_LO_TO(cap_lo) + 1; ctrlr->ready_timeout_in_ms = to * 500; + timeout_period = NVME_ADMIN_TIMEOUT_PERIOD; + TUNABLE_INT_FETCH("hw.nvme.admin_timeout_period", &timeout_period); + timeout_period = min(timeout_period, NVME_MAX_TIMEOUT_PERIOD); + timeout_period = max(timeout_period, NVME_MIN_TIMEOUT_PERIOD); + ctrlr->admin_timeout_period = timeout_period; + timeout_period = NVME_DEFAULT_TIMEOUT_PERIOD; TUNABLE_INT_FETCH("hw.nvme.timeout_period", &timeout_period); timeout_period = min(timeout_period, NVME_MAX_TIMEOUT_PERIOD); diff --git a/sys/dev/nvme/nvme_private.h b/sys/dev/nvme/nvme_private.h index 496bd8229e0..72797d30a3b 100644 --- a/sys/dev/nvme/nvme_private.h +++ b/sys/dev/nvme/nvme_private.h @@ -86,6 +86,7 @@ MALLOC_DECLARE(M_NVME); #define NVME_MAX_CONSUMERS (2) #define NVME_MAX_ASYNC_EVENTS (8) +#define NVME_ADMIN_TIMEOUT_PERIOD (60) /* in seconds */ #define NVME_DEFAULT_TIMEOUT_PERIOD (30) /* in seconds */ #define NVME_MIN_TIMEOUT_PERIOD (5) #define NVME_MAX_TIMEOUT_PERIOD (120) @@ -281,6 +282,7 @@ struct nvme_controller { uint32_t int_coal_threshold; /** timeout period in seconds */ + uint32_t admin_timeout_period; uint32_t timeout_period; /** doorbell stride */ diff --git a/sys/dev/nvme/nvme_qpair.c b/sys/dev/nvme/nvme_qpair.c index cd0057f444b..2e9af4310a1 100644 --- a/sys/dev/nvme/nvme_qpair.c +++ b/sys/dev/nvme/nvme_qpair.c @@ -1158,6 +1158,8 @@ nvme_qpair_submit_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr) if (req->timeout) { if (req->cb_fn == nvme_completion_poll_cb) timeout = 1; + else if (qpair->id == 0) + timeout = ctrlr->admin_timeout_period; else timeout = ctrlr->timeout_period; tr->deadline = getsbinuptime() + timeout * SBT_1S; diff --git a/sys/dev/nvme/nvme_sysctl.c b/sys/dev/nvme/nvme_sysctl.c index ac0d507e233..d6452a2e549 100644 --- a/sys/dev/nvme/nvme_sysctl.c +++ b/sys/dev/nvme/nvme_sysctl.c @@ -132,8 +132,8 @@ nvme_sysctl_int_coal_threshold(SYSCTL_HANDLER_ARGS) static int nvme_sysctl_timeout_period(SYSCTL_HANDLER_ARGS) { - struct nvme_controller *ctrlr = arg1; - uint32_t newval = ctrlr->timeout_period; + uint32_t *ptr = arg1; + uint32_t newval = *ptr; int error = sysctl_handle_int(oidp, &newval, 0, req); if (error || (req->newptr == NULL)) @@ -143,7 +143,7 @@ nvme_sysctl_timeout_period(SYSCTL_HANDLER_ARGS) newval < NVME_MIN_TIMEOUT_PERIOD) { return (EINVAL); } else { - ctrlr->timeout_period = newval; + *ptr = newval; } return (0); @@ -352,10 +352,15 @@ nvme_sysctl_initialize_ctrlr(struct nvme_controller *ctrlr) nvme_sysctl_int_coal_threshold, "IU", "Interrupt coalescing threshold"); + SYSCTL_ADD_PROC(ctrlr_ctx, ctrlr_list, OID_AUTO, + "admin_timeout_period", CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, + &ctrlr->admin_timeout_period, 0, nvme_sysctl_timeout_period, "IU", + "Timeout period for Admin queue (in seconds)"); + SYSCTL_ADD_PROC(ctrlr_ctx, ctrlr_list, OID_AUTO, "timeout_period", CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, - ctrlr, 0, nvme_sysctl_timeout_period, "IU", - "Timeout period (in seconds)"); + &ctrlr->timeout_period, 0, nvme_sysctl_timeout_period, "IU", + "Timeout period for I/O queues (in seconds)"); SYSCTL_ADD_PROC(ctrlr_ctx, ctrlr_list, OID_AUTO, "num_cmds", CTLTYPE_S64 | CTLFLAG_RD | CTLFLAG_MPSAFE,