From 0e4cd4a2e0eb515cd0e3f0e935704388c46030f6 Mon Sep 17 00:00:00 2001
From: Navdeep Parhar <np@FreeBSD.org>
Date: Tue, 16 Jun 2015 12:36:29 +0000
Subject: [PATCH] cxgbe(4): Add the ability to dump mailbox commands and
 replies.  It is enabled/disabled via bit 0 of adapter->debug_flags (which is
 available at dev.t5nex.<n>.debug_flags).

MFC after:	1 week
---
 sys/dev/cxgbe/adapter.h      | 22 ++++++++++++++++++++++
 sys/dev/cxgbe/common/t4_hw.c |  4 ++++
 sys/dev/cxgbe/t4_main.c      |  6 ++++++
 3 files changed, 32 insertions(+)

diff --git a/sys/dev/cxgbe/adapter.h b/sys/dev/cxgbe/adapter.h
index 2655dae1c0f..c1782971424 100644
--- a/sys/dev/cxgbe/adapter.h
+++ b/sys/dev/cxgbe/adapter.h
@@ -206,6 +206,9 @@ enum {
 	INTR_OFLD_RXQ	= (1 << 5),	/* All TOE rxq's take interrupts */
 	INTR_NM_RXQ	= (1 << 6),	/* All netmap rxq's take interrupts */
 	INTR_ALL	= (INTR_RXQ | INTR_OFLD_RXQ | INTR_NM_RXQ),
+
+	/* adapter debug_flags */
+	DF_DUMP_MBOX	= (1 << 0),
 };
 
 #define IS_DOOMED(pi)	((pi)->flags & DOOMED)
@@ -762,6 +765,7 @@ struct adapter {
 	int active_ulds;	/* ULDs activated on this adapter */
 #endif
 	int flags;
+	int debug_flags;
 
 	char ifp_lockname[16];
 	struct mtx ifp_lock;
@@ -846,6 +850,24 @@ struct adapter {
 #define TXQ_LOCK_ASSERT_OWNED(txq)	EQ_LOCK_ASSERT_OWNED(&(txq)->eq)
 #define TXQ_LOCK_ASSERT_NOTOWNED(txq)	EQ_LOCK_ASSERT_NOTOWNED(&(txq)->eq)
 
+#define CH_DUMP_MBOX(sc, mbox, data_reg) \
+	do { \
+		if (sc->debug_flags & DF_DUMP_MBOX) { \
+			log(LOG_NOTICE, \
+			    "%s mbox %u: %016llx %016llx %016llx %016llx " \
+			    "%016llx %016llx %016llx %016llx\n", \
+			    device_get_nameunit(sc->dev), mbox, \
+			    (unsigned long long)t4_read_reg64(sc, data_reg), \
+			    (unsigned long long)t4_read_reg64(sc, data_reg + 8), \
+			    (unsigned long long)t4_read_reg64(sc, data_reg + 16), \
+			    (unsigned long long)t4_read_reg64(sc, data_reg + 24), \
+			    (unsigned long long)t4_read_reg64(sc, data_reg + 32), \
+			    (unsigned long long)t4_read_reg64(sc, data_reg + 40), \
+			    (unsigned long long)t4_read_reg64(sc, data_reg + 48), \
+			    (unsigned long long)t4_read_reg64(sc, data_reg + 56)); \
+		} \
+	} while (0)
+
 #define for_each_txq(pi, iter, q) \
 	for (q = &pi->adapter->sge.txq[pi->first_txq], iter = 0; \
 	    iter < pi->ntxq; ++iter, ++q)
diff --git a/sys/dev/cxgbe/common/t4_hw.c b/sys/dev/cxgbe/common/t4_hw.c
index e2efb782e70..d140276722c 100644
--- a/sys/dev/cxgbe/common/t4_hw.c
+++ b/sys/dev/cxgbe/common/t4_hw.c
@@ -262,6 +262,8 @@ int t4_wr_mbox_meat(struct adapter *adap, int mbox, const void *cmd, int size,
 	for (i = 0; i < size; i += 8, p++)
 		t4_write_reg64(adap, data_reg + i, be64_to_cpu(*p));
 
+	CH_DUMP_MBOX(adap, mbox, data_reg);
+
 	t4_write_reg(adap, ctl_reg, F_MBMSGVALID | V_MBOWNER(X_MBOWNER_FW));
 	t4_read_reg(adap, ctl_reg);          /* flush write */
 
@@ -287,6 +289,8 @@ int t4_wr_mbox_meat(struct adapter *adap, int mbox, const void *cmd, int size,
 				continue;
 			}
 
+			CH_DUMP_MBOX(adap, mbox, data_reg);
+
 			res = t4_read_reg64(adap, data_reg);
 			if (G_FW_CMD_OP(res >> 32) == FW_DEBUG_CMD) {
 				fw_asrt(adap, data_reg);
diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c
index b164cce731e..5c8805967f0 100644
--- a/sys/dev/cxgbe/t4_main.c
+++ b/sys/dev/cxgbe/t4_main.c
@@ -585,6 +585,9 @@ t4_attach(device_t dev)
 
 	sc = device_get_softc(dev);
 	sc->dev = dev;
+#ifdef INVARIANTS
+	sc->debug_flags = DF_DUMP_MBOX;
+#endif
 
 	pci_enable_busmaster(dev);
 	if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
@@ -4603,6 +4606,9 @@ t4_sysctls(struct adapter *sc)
 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
 	    &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
 
+	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "debug_flags", CTLFLAG_RW,
+	    &sc->debug_flags, 0, "flags to enable runtime debugging");
+
 #ifdef SBUF_DRAIN
 	/*
 	 * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.