diff --git a/sys/kern/subr_trap.c b/sys/kern/subr_trap.c index 6160b3fcdf3..42ce9e7988b 100644 --- a/sys/kern/subr_trap.c +++ b/sys/kern/subr_trap.c @@ -76,6 +76,8 @@ #include #endif +void (*tcp_hpts_softclock)(void); + /* * Define the code needed before returning to user mode, for trap and * syscall. @@ -127,16 +129,18 @@ userret(struct thread *td, struct trapframe *frame) if (PMC_THREAD_HAS_SAMPLES(td)) PMC_CALL_HOOK(td, PMC_FN_THR_USERRET, NULL); #endif -#ifdef TCPHPTS /* - * @gallatin is adament that this needs to go here, I - * am not so sure. Running hpts is a lot like - * a lro_flush() that happens while a user process - * is running. But he may know best so I will go - * with his view of accounting. :-) + * Calling tcp_hpts_softclock() here allows us to avoid frequent, + * expensive callouts that trash the cache and lead to a much higher + * number of interrupts and context switches. Testing on busy web + * servers at Netflix has shown that this improves CPU use by 7% over + * relying only on callouts to drive HPTS, and also results in idle + * power savings on mostly idle servers. + * This was inspired by the paper "Soft Timers: Efficient Microsecond + * Software Timer Support for Network Processing" + * by Mohit Aron and Peter Druschel. */ - tcp_run_hpts(); -#endif + tcp_hpts_softclock(); /* * Let the scheduler adjust our priority etc. */ diff --git a/sys/netinet/tcp_hpts.h b/sys/netinet/tcp_hpts.h index 8ca21daf60d..7eb1b2e08cb 100644 --- a/sys/netinet/tcp_hpts.h +++ b/sys/netinet/tcp_hpts.h @@ -152,7 +152,6 @@ void __tcp_set_hpts(struct tcpcb *tp, int32_t line); void tcp_set_inp_to_drop(struct inpcb *inp, uint16_t reason); -extern void (*tcp_hpts_softclock)(void); void tcp_lro_hpts_init(void); extern int32_t tcp_min_hptsi_time; diff --git a/sys/netinet/tcp_lro.c b/sys/netinet/tcp_lro.c index 255e543ae21..921d28f8251 100644 --- a/sys/netinet/tcp_lro.c +++ b/sys/netinet/tcp_lro.c @@ -89,7 +89,6 @@ SYSCTL_NODE(_net_inet_tcp, OID_AUTO, lro, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, long tcplro_stacks_wanting_mbufq; int (*tcp_lro_flush_tcphpts)(struct lro_ctrl *lc, struct lro_entry *le); -void (*tcp_hpts_softclock)(void); counter_u64_t tcp_inp_lro_direct_queue; counter_u64_t tcp_inp_lro_wokeup_queue; @@ -1262,8 +1261,7 @@ tcp_lro_flush_all(struct lro_ctrl *lc) done: /* flush active streams */ tcp_lro_rx_done(lc); - if (tcp_hpts_softclock != NULL) - tcp_hpts_softclock(); + tcp_hpts_softclock(); lc->lro_mbuf_count = 0; } diff --git a/sys/sys/systm.h b/sys/sys/systm.h index 2367a7880cf..0e90b6062b7 100644 --- a/sys/sys/systm.h +++ b/sys/sys/systm.h @@ -380,6 +380,12 @@ void cpu_et_frequency(struct eventtimer *et, uint64_t newfreq); extern int cpu_disable_c2_sleep; extern int cpu_disable_c3_sleep; +extern void (*tcp_hpts_softclock)(void); +#define tcp_hpts_softclock() do { \ + if (tcp_hpts_softclock != NULL) \ + tcp_hpts_softclock(); \ +} while (0) + char *kern_getenv(const char *name); void freeenv(char *env); int getenv_int(const char *name, int *data);