diff --git a/sys/dev/cxgbe/offload.h b/sys/dev/cxgbe/offload.h
index 466b61c0151..968902cb10d 100644
--- a/sys/dev/cxgbe/offload.h
+++ b/sys/dev/cxgbe/offload.h
@@ -225,6 +225,7 @@ struct tom_tunables {
 	int ddp;
 	int rx_coalesce;
 	int tls;
+	int tls_rx_timeout;
 	int *tls_rx_ports;
 	int num_tls_rx_ports;
 	int tx_align;
diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c
index 03b33116ce8..32a229bed52 100644
--- a/sys/dev/cxgbe/t4_main.c
+++ b/sys/dev/cxgbe/t4_main.c
@@ -403,6 +403,11 @@ SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 14, CTLFLAG_RDTUN,
     &t4_toe_rexmt_backoff[14], 0, "");
 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 15, CTLFLAG_RDTUN,
     &t4_toe_rexmt_backoff[15], 0, "");
+
+static int t4_toe_tls_rx_timeout = 5;
+SYSCTL_INT(_hw_cxgbe_toe, OID_AUTO, tls_rx_timeout, CTLFLAG_RDTUN,
+    &t4_toe_tls_rx_timeout, 0,
+    "Timeout in seconds to downgrade TLS sockets to plain TOE");
 #endif
 
 #ifdef DEV_NETMAP
@@ -786,6 +791,7 @@ static int sysctl_cpus(SYSCTL_HANDLER_ARGS);
 #ifdef TCP_OFFLOAD
 static int sysctl_tls(SYSCTL_HANDLER_ARGS);
 static int sysctl_tls_rx_ports(SYSCTL_HANDLER_ARGS);
+static int sysctl_tls_rx_timeout(SYSCTL_HANDLER_ARGS);
 static int sysctl_tp_tick(SYSCTL_HANDLER_ARGS);
 static int sysctl_tp_dack_timer(SYSCTL_HANDLER_ARGS);
 static int sysctl_tp_timer(SYSCTL_HANDLER_ARGS);
@@ -6789,6 +6795,12 @@ t4_sysctls(struct adapter *sc)
 		    sysctl_tls_rx_ports, "I",
 		    "TCP ports that use inline TLS+TOE RX");
 
+		sc->tt.tls_rx_timeout = t4_toe_tls_rx_timeout;
+		SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tls_rx_timeout",
+		    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0,
+		    sysctl_tls_rx_timeout, "I",
+		    "Timeout in seconds to downgrade TLS sockets to plain TOE");
+
 		sc->tt.tx_align = -1;
 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align",
 		    CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload");
@@ -10046,6 +10058,29 @@ sysctl_tls_rx_ports(SYSCTL_HANDLER_ARGS)
 	return (rc);
 }
 
+static int
+sysctl_tls_rx_timeout(SYSCTL_HANDLER_ARGS)
+{
+	struct adapter *sc = arg1;
+	int v, rc;
+
+	v = sc->tt.tls_rx_timeout;
+	rc = sysctl_handle_int(oidp, &v, 0, req);
+	if (rc != 0 || req->newptr == NULL)
+		return (rc);
+
+	if (v < 0)
+		return (EINVAL);
+
+	if (v != 0 && !(sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS))
+		return (ENOTSUP);
+
+	sc->tt.tls_rx_timeout = v;
+
+	return (0);
+
+}
+
 static void
 unit_conv(char *buf, size_t len, u_int val, u_int factor)
 {
@@ -11287,6 +11322,9 @@ tweak_tunables(void)
 
 	if (t4_pktc_idx_ofld < -1 || t4_pktc_idx_ofld >= SGE_NCOUNTERS)
 		t4_pktc_idx_ofld = PKTC_IDX_OFLD;
+
+	if (t4_toe_tls_rx_timeout < 0)
+		t4_toe_tls_rx_timeout = 0;
 #else
 	if (t4_rdmacaps_allowed == -1)
 		t4_rdmacaps_allowed = 0;
diff --git a/sys/dev/cxgbe/tom/t4_connect.c b/sys/dev/cxgbe/tom/t4_connect.c
index b6b523637da..c285b6fc41f 100644
--- a/sys/dev/cxgbe/tom/t4_connect.c
+++ b/sys/dev/cxgbe/tom/t4_connect.c
@@ -105,9 +105,6 @@ do_act_establish(struct sge_iq *iq, const struct rss_header *rss,
 	inp->inp_flowtype = M_HASHTYPE_OPAQUE;
 	inp->inp_flowid = tid;
 
-	if (ulp_mode(toep) == ULP_MODE_TLS)
-		tls_establish(toep);
-
 done:
 	INP_WUNLOCK(inp);
 	CURVNET_RESTORE();
diff --git a/sys/dev/cxgbe/tom/t4_cpl_io.c b/sys/dev/cxgbe/tom/t4_cpl_io.c
index c8bc0fda694..f1d4ce6825c 100644
--- a/sys/dev/cxgbe/tom/t4_cpl_io.c
+++ b/sys/dev/cxgbe/tom/t4_cpl_io.c
@@ -392,6 +392,9 @@ make_established(struct toepcb *toep, uint32_t iss, uint32_t irs, uint16_t opt)
 	send_flowc_wr(toep, tp);
 
 	soisconnected(so);
+
+	if (ulp_mode(toep) == ULP_MODE_TLS)
+		tls_establish(toep);
 }
 
 int
diff --git a/sys/dev/cxgbe/tom/t4_tls.c b/sys/dev/cxgbe/tom/t4_tls.c
index 36e1c66a3f2..5cd043a6853 100644
--- a/sys/dev/cxgbe/tom/t4_tls.c
+++ b/sys/dev/cxgbe/tom/t4_tls.c
@@ -64,14 +64,6 @@ __FBSDID("$FreeBSD$");
  */
 #define	tls_tcp_seq	PH_loc.thirtytwo[0]
 
-/*
- * Handshake lock used for the handshake timer.  Having a global lock
- * is perhaps not ideal, but it avoids having to use callout_drain()
- * in tls_uninit_toep() which can't block.  Also, the timer shouldn't
- * actually fire for most connections.
- */
-static struct mtx tls_handshake_lock;
-
 static void
 t4_set_tls_tcb_field(struct toepcb *toep, uint16_t word, uint64_t mask,
     uint64_t val)
@@ -149,7 +141,7 @@ tls_clr_ofld_mode(struct toepcb *toep)
 	    V_TCB_ULP_TYPE(M_TCB_ULP_TYPE), V_TCB_ULP_TYPE(ULP_MODE_NONE));
 	t4_clear_rx_quiesce(toep);
 
-	toep->flags &= ~TPF_FORCE_CREDITS;
+	toep->flags &= ~(TPF_FORCE_CREDITS | TPF_TLS_ESTABLISHED);
 	toep->params.ulp_mode = ULP_MODE_NONE;
 }
 
@@ -747,6 +739,25 @@ tls_send_handshake_ack(void *arg)
 	struct tls_ofld_info *tls_ofld = &toep->tls;
 	struct adapter *sc = td_adapter(toep->td);
 
+	/*
+	 * If this connection has timed out without receiving more
+	 * data, downgrade to plain TOE mode and don't re-arm the
+	 * timer.
+	 */
+	if (sc->tt.tls_rx_timeout != 0) {
+		struct inpcb *inp;
+		struct tcpcb *tp;
+
+		inp = toep->inp;
+		tp = intotcpcb(inp);
+		if ((ticks - tp->t_rcvtime) >= sc->tt.tls_rx_timeout) {
+			CTR2(KTR_CXGBE, "%s: tid %d clr_ofld_mode", __func__,
+			    toep->tid);
+			tls_clr_ofld_mode(toep);
+			return;
+		}
+	}
+
 	/*
 	 * XXX: Does not have the t4_get_tcb() checks to refine the
 	 * workaround.
@@ -762,10 +773,9 @@ tls_start_handshake_timer(struct toepcb *toep)
 {
 	struct tls_ofld_info *tls_ofld = &toep->tls;
 
-	mtx_lock(&tls_handshake_lock);
+	INP_WLOCK_ASSERT(toep->inp);
 	callout_reset(&tls_ofld->handshake_timer, TLS_SRV_HELLO_BKOFF_TM * hz,
 	    tls_send_handshake_ack, toep);
-	mtx_unlock(&tls_handshake_lock);
 }
 
 void
@@ -773,9 +783,8 @@ tls_stop_handshake_timer(struct toepcb *toep)
 {
 	struct tls_ofld_info *tls_ofld = &toep->tls;
 
-	mtx_lock(&tls_handshake_lock);
+	INP_WLOCK_ASSERT(toep->inp);
 	callout_stop(&tls_ofld->handshake_timer);
-	mtx_unlock(&tls_handshake_lock);
 }
 
 int
@@ -1129,9 +1138,6 @@ tls_init_toep(struct toepcb *toep)
 	tls_ofld->key_location = TLS_SFO_WR_CONTEXTLOC_DDR;
 	tls_ofld->rx_key_addr = -1;
 	tls_ofld->tx_key_addr = -1;
-	if (ulp_mode(toep) == ULP_MODE_TLS)
-		callout_init_mtx(&tls_ofld->handshake_timer,
-		    &tls_handshake_lock, 0);
 }
 
 void
@@ -1149,17 +1155,27 @@ tls_establish(struct toepcb *toep)
 	t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, V_TCB_ULP_RAW(M_TCB_ULP_RAW),
 	    V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1)));
 
-	toep->flags |= TPF_FORCE_CREDITS;
+	toep->flags |= TPF_FORCE_CREDITS | TPF_TLS_ESTABLISHED;
 
+	callout_init_rw(&toep->tls.handshake_timer, &toep->inp->inp_lock, 0);
 	tls_start_handshake_timer(toep);
 }
 
+void
+tls_detach(struct toepcb *toep)
+{
+
+	if (toep->flags & TPF_TLS_ESTABLISHED) {
+		tls_stop_handshake_timer(toep);
+		toep->flags &= ~TPF_TLS_ESTABLISHED;
+	}
+}
+
 void
 tls_uninit_toep(struct toepcb *toep)
 {
 
-	if (ulp_mode(toep) == ULP_MODE_TLS)
-		tls_stop_handshake_timer(toep);
+	MPASS((toep->flags & TPF_TLS_ESTABLISHED) == 0);
 	clear_tls_keyid(toep);
 }
 
@@ -2383,7 +2399,6 @@ void
 t4_tls_mod_load(void)
 {
 
-	mtx_init(&tls_handshake_lock, "t4tls handshake", NULL, MTX_DEF);
 	t4_register_cpl_handler(CPL_TLS_DATA, do_tls_data);
 	t4_register_cpl_handler(CPL_RX_TLS_CMP, do_rx_tls_cmp);
 }
@@ -2394,6 +2409,5 @@ t4_tls_mod_unload(void)
 
 	t4_register_cpl_handler(CPL_TLS_DATA, NULL);
 	t4_register_cpl_handler(CPL_RX_TLS_CMP, NULL);
-	mtx_destroy(&tls_handshake_lock);
 }
 #endif	/* TCP_OFFLOAD */
diff --git a/sys/dev/cxgbe/tom/t4_tom.c b/sys/dev/cxgbe/tom/t4_tom.c
index 705a2175f93..0f0b4d7ee5b 100644
--- a/sys/dev/cxgbe/tom/t4_tom.c
+++ b/sys/dev/cxgbe/tom/t4_tom.c
@@ -382,6 +382,9 @@ t4_pcb_detach(struct toedev *tod __unused, struct tcpcb *tp)
 	}
 #endif
 
+	if (ulp_mode(toep) == ULP_MODE_TLS)
+		tls_detach(toep);
+
 	tp->tod = NULL;
 	tp->t_toe = NULL;
 	tp->t_flags &= ~TF_TOE;
@@ -845,6 +848,8 @@ final_cpl_received(struct toepcb *toep)
 
 	if (ulp_mode(toep) == ULP_MODE_TCPDDP)
 		release_ddp_resources(toep);
+	else if (ulp_mode(toep) == ULP_MODE_TLS)
+		tls_detach(toep);
 	toep->inp = NULL;
 	toep->flags &= ~TPF_CPL_PENDING;
 	mbufq_drain(&toep->ulp_pdu_reclaimq);
diff --git a/sys/dev/cxgbe/tom/t4_tom.h b/sys/dev/cxgbe/tom/t4_tom.h
index be9f3e90e0a..41187aa27e2 100644
--- a/sys/dev/cxgbe/tom/t4_tom.h
+++ b/sys/dev/cxgbe/tom/t4_tom.h
@@ -75,6 +75,7 @@ enum {
 	TPF_KTLS           = (1 << 11), /* send TLS records from KTLS */
 	TPF_INITIALIZED    = (1 << 12), /* init_toepcb has been called */
 	TPF_TLS_RECEIVE	   = (1 << 13), /* should receive TLS records */
+	TPF_TLS_ESTABLISHED = (1 << 14), /* TLS handshake timer initialized */
 };
 
 enum {
@@ -448,6 +449,7 @@ void t4_push_tls_records(struct adapter *, struct toepcb *, int);
 void t4_push_ktls(struct adapter *, struct toepcb *, int);
 void t4_tls_mod_load(void);
 void t4_tls_mod_unload(void);
+void tls_detach(struct toepcb *);
 void tls_establish(struct toepcb *);
 void tls_init_toep(struct toepcb *);
 int tls_rx_key(struct toepcb *);