From 1bbc06b85b06630f38959fd9b42f879dfbc52d96 Mon Sep 17 00:00:00 2001 From: Conrad Meyer Date: Mon, 29 Aug 2016 20:46:33 +0000 Subject: [PATCH] ioat(4): Don't "complete" DMA descriptors prematurely In r304602, I mistakenly removed the ioat_process_events check that we weren't processing events before the hardware had completed the descriptor ("last_seen"). Reinstate that logic. Keep the defensive loop condition and additionally make sure we've actually completed a descriptor before blindly chasing the ring around. In reset, queue and finish the startup command before allowing any event processing or submission to occur. Avoid potential missed callouts by requeueing the poll later. --- sys/dev/ioat/ioat.c | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/sys/dev/ioat/ioat.c b/sys/dev/ioat/ioat.c index 8443e3a9260..f98e82d1ef8 100644 --- a/sys/dev/ioat/ioat.c +++ b/sys/dev/ioat/ioat.c @@ -683,7 +683,16 @@ ioat_process_events(struct ioat_softc *ioat) __func__, ioat->chan_idx, comp_update, ioat->last_seen); status = comp_update & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK; - while (ioat_get_active(ioat) > 0) { + if (status == ioat->last_seen) { + /* + * If we landed in process_events and nothing has been + * completed, check for a timeout due to channel halt. + */ + goto out; + } + + desc = ioat_get_ring_entry(ioat, ioat->tail - 1); + while (desc->hw_desc_bus_addr != status && ioat_get_active(ioat) > 0) { desc = ioat_get_ring_entry(ioat, ioat->tail); dmadesc = &desc->bus_dmadesc; CTR4(KTR_IOAT, "channel=%u completing desc %u ok cb %p(%p)", @@ -695,8 +704,6 @@ ioat_process_events(struct ioat_softc *ioat) completed++; ioat->tail++; - if (desc->hw_desc_bus_addr == status) - break; } if (completed != 0) { @@ -704,6 +711,7 @@ ioat_process_events(struct ioat_softc *ioat) ioat->stats.descriptors_processed += completed; } +out: ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN); /* Perform a racy check first; only take the locks if it passes. */ @@ -1913,19 +1921,17 @@ ioat_reset_hw(struct ioat_softc *ioat) error = 0; out: + /* Enqueues a null operation and ensures it completes. */ + if (error == 0) + error = ioat_start_channel(ioat); + /* * Resume completions now that ring state is consistent. - * ioat_start_channel will add a pending completion and if we are still - * blocking completions, we may livelock. */ mtx_lock(&ioat->cleanup_lock); ioat->resetting_cleanup = FALSE; mtx_unlock(&ioat->cleanup_lock); - /* Enqueues a null operation and ensures it completes. */ - if (error == 0) - error = ioat_start_channel(ioat); - /* Unblock submission of new work */ mtx_lock(IOAT_REFLK); ioat->quiescing = FALSE; @@ -1933,6 +1939,10 @@ out: ioat->resetting = FALSE; wakeup(&ioat->resetting); + + if (ioat->is_completion_pending) + callout_reset(&ioat->poll_timer, 1, ioat_poll_timer_callback, + ioat); mtx_unlock(IOAT_REFLK); return (error);