diff --git a/sys/netinet/tcp_input.c b/sys/netinet/tcp_input.c
index 87a52969221..77c7beac858 100644
--- a/sys/netinet/tcp_input.c
+++ b/sys/netinet/tcp_input.c
@@ -1565,8 +1565,6 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so,
 	 * validation to ignore broken/spoofed segs.
 	 */
 	tp->t_rcvtime = ticks;
-	if (TCPS_HAVEESTABLISHED(tp->t_state))
-		tcp_timer_activate(tp, TT_KEEP, TP_KEEPIDLE(tp));
 
 	/*
 	 * Scale up the window into a 32-bit value.
diff --git a/sys/netinet/tcp_stacks/fastpath.c b/sys/netinet/tcp_stacks/fastpath.c
index 4912854749e..a31a28e70f6 100644
--- a/sys/netinet/tcp_stacks/fastpath.c
+++ b/sys/netinet/tcp_stacks/fastpath.c
@@ -1819,8 +1819,6 @@ tcp_do_segment_fastslow(struct mbuf *m, struct tcphdr *th, struct socket *so,
 	 * validation to ignore broken/spoofed segs.
 	 */
 	tp->t_rcvtime = ticks;
-	if (TCPS_HAVEESTABLISHED(tp->t_state))
-		tcp_timer_activate(tp, TT_KEEP, TP_KEEPIDLE(tp));
 
 	/*
 	 * Unscale the window into a 32-bit value.
@@ -2266,8 +2264,6 @@ tcp_do_segment_fastack(struct mbuf *m, struct tcphdr *th, struct socket *so,
 	 * validation to ignore broken/spoofed segs.
 	 */
 	tp->t_rcvtime = ticks;
-	if (TCPS_HAVEESTABLISHED(tp->t_state))
-		tcp_timer_activate(tp, TT_KEEP, TP_KEEPIDLE(tp));
 
 	/*
 	 * Unscale the window into a 32-bit value.
diff --git a/sys/netinet/tcp_timer.c b/sys/netinet/tcp_timer.c
index 5f231c07978..f9f5c05fdce 100644
--- a/sys/netinet/tcp_timer.c
+++ b/sys/netinet/tcp_timer.c
@@ -468,6 +468,26 @@ tcp_timer_keep(void *xtp)
 	}
 	KASSERT((tp->t_timers->tt_flags & TT_STOPPED) == 0,
 		("%s: tp %p tcpcb can't be stopped here", __func__, tp));
+
+	/*
+	 * Because we don't regularly reset the keepalive callout in
+	 * the ESTABLISHED state, it may be that we don't actually need
+	 * to send a keepalive yet. If that occurs, schedule another
+	 * call for the next time the keepalive timer might expire.
+	 */
+	if (TCPS_HAVEESTABLISHED(tp->t_state)) {
+		u_int idletime;
+
+		idletime = ticks - tp->t_rcvtime;
+		if (idletime < TP_KEEPIDLE(tp)) {
+			callout_reset(&tp->t_timers->tt_keep,
+			    TP_KEEPIDLE(tp) - idletime, tcp_timer_keep, tp);
+			INP_WUNLOCK(inp);
+			CURVNET_RESTORE();
+			return;
+		}
+	}
+
 	/*
 	 * Keep-alive timer went off; send something
 	 * or drop connection if idle for too long.