From eaa6dfbcc201545be96e1cf899c83923300c8d48 Mon Sep 17 00:00:00 2001 From: Robert Watson Date: Tue, 1 Aug 2006 10:30:26 +0000 Subject: [PATCH] Reimplement socket buffer tear-down in sofree(): as the socket is no longer referenced by other threads (hence our freeing it), we don't need to set the can't send and can't receive flags, wake up the consumers, perform two levels of locking, etc. Implement a fast-path teardown, sbdestroy(), which flushes and releases each socket buffer. A manual dom_dispose of the receive buffer is still required explicitly to GC any in-flight file descriptors, etc, before flushing the buffer. This results in a 9% UP performance improvement and 16% SMP performance improvement on a tight loop of socket();close(); in micro-benchmarking, but will likely also affect CPU-bound macro-benchmark performance. --- sys/kern/uipc_sockbuf.c | 66 +++++++++++++++++++++++++++++++++-------- sys/kern/uipc_socket.c | 36 +++++++++++++--------- sys/sys/socketvar.h | 1 + 3 files changed, 76 insertions(+), 27 deletions(-) diff --git a/sys/kern/uipc_sockbuf.c b/sys/kern/uipc_sockbuf.c index e26627297b6..72c54ac780b 100644 --- a/sys/kern/uipc_sockbuf.c +++ b/sys/kern/uipc_sockbuf.c @@ -64,6 +64,10 @@ static u_long sb_max_adj = static u_long sb_efficiency = 8; /* parameter for sbreserve() */ +static void sbdrop_internal(register struct sockbuf *sb, register int len); +static void sbflush_internal(register struct sockbuf *sb); +static void sbrelease_internal(struct sockbuf *sb, struct socket *so); + /* * Socantsendmore indicates that no more data will be sent on the * socket; it would normally be applied to a socket when the user @@ -331,6 +335,18 @@ sbreserve(sb, cc, so, td) /* * Free mbufs held by a socket, and reserved mbuf space. */ +static void +sbrelease_internal(sb, so) + struct sockbuf *sb; + struct socket *so; +{ + + sbflush_internal(sb); + (void)chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, 0, + RLIM_INFINITY); + sb->sb_mbmax = 0; +} + void sbrelease_locked(sb, so) struct sockbuf *sb; @@ -339,10 +355,7 @@ sbrelease_locked(sb, so) SOCKBUF_LOCK_ASSERT(sb); - sbflush_locked(sb); - (void)chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, 0, - RLIM_INFINITY); - sb->sb_mbmax = 0; + sbrelease_internal(sb, so); } void @@ -355,6 +368,17 @@ sbrelease(sb, so) sbrelease_locked(sb, so); SOCKBUF_UNLOCK(sb); } + +void +sbdestroy(sb, so) + struct sockbuf *sb; + struct socket *so; +{ + + sbrelease_internal(sb, so); +} + + /* * Routines to add and remove * data from an mbuf queue. @@ -823,13 +847,11 @@ sbcompress(sb, m, n) * Free all mbufs in a sockbuf. * Check that all resources are reclaimed. */ -void -sbflush_locked(sb) +static void +sbflush_internal(sb) register struct sockbuf *sb; { - SOCKBUF_LOCK_ASSERT(sb); - if (sb->sb_flags & SB_LOCK) panic("sbflush_locked: locked"); while (sb->sb_mbcnt) { @@ -839,12 +861,21 @@ sbflush_locked(sb) */ if (!sb->sb_cc && (sb->sb_mb == NULL || sb->sb_mb->m_len)) break; - sbdrop_locked(sb, (int)sb->sb_cc); + sbdrop_internal(sb, (int)sb->sb_cc); } if (sb->sb_cc || sb->sb_mb || sb->sb_mbcnt) panic("sbflush_locked: cc %u || mb %p || mbcnt %u", sb->sb_cc, (void *)sb->sb_mb, sb->sb_mbcnt); } +void +sbflush_locked(sb) + register struct sockbuf *sb; +{ + + SOCKBUF_LOCK_ASSERT(sb); + sbflush_internal(sb); +} + void sbflush(sb) register struct sockbuf *sb; @@ -858,16 +889,14 @@ sbflush(sb) /* * Drop data from (the front of) a sockbuf. */ -void -sbdrop_locked(sb, len) +static void +sbdrop_internal(sb, len) register struct sockbuf *sb; register int len; { register struct mbuf *m; struct mbuf *next; - SOCKBUF_LOCK_ASSERT(sb); - next = (m = sb->sb_mb) ? m->m_nextpkt : 0; while (len > 0) { if (m == 0) { @@ -915,6 +944,17 @@ sbdrop_locked(sb, len) /* * Drop data from (the front of) a sockbuf. */ +void +sbdrop_locked(sb, len) + register struct sockbuf *sb; + register int len; +{ + + SOCKBUF_LOCK_ASSERT(sb); + + sbdrop_internal(sb, len); +} + void sbdrop(sb, len) register struct sockbuf *sb; diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c index b92e0850ca6..b5af0819fbc 100644 --- a/sys/kern/uipc_socket.c +++ b/sys/kern/uipc_socket.c @@ -555,6 +555,7 @@ void sofree(so) struct socket *so; { + struct protosw *pr = so->so_proto; struct socket *head; ACCEPT_LOCK_ASSERT(); @@ -588,24 +589,31 @@ sofree(so) SOCK_UNLOCK(so); ACCEPT_UNLOCK(); - SOCKBUF_LOCK(&so->so_snd); - so->so_snd.sb_flags |= SB_NOINTR; - (void)sblock(&so->so_snd, M_WAITOK); /* - * socantsendmore_locked() drops the socket buffer mutex so that it - * can safely perform wakeups. Re-acquire the mutex before - * continuing. + * From this point on, we assume that no other references to this + * socket exist anywhere else in the stack. Therefore, no locks need + * to be acquired or held. + * + * We used to do a lot of socket buffer and socket locking here, as + * well as invoke sorflush() and perform wakeups. The direct call to + * dom_dispose() and sbrelease_internal() are an inlining of what was + * necessary from sorflush(). + * + * Notice that the socket buffer and kqueue state are torn down + * before calling pru_detach. This means that protocols shold not + * assume they can perform socket wakeups, etc, in their detach + * code. */ - socantsendmore_locked(so); - SOCKBUF_LOCK(&so->so_snd); - sbunlock(&so->so_snd); - sbrelease_locked(&so->so_snd, so); - SOCKBUF_UNLOCK(&so->so_snd); - sorflush(so); + KASSERT((so->so_snd.sb_flags & SB_LOCK) == 0, ("sofree: snd sblock")); + KASSERT((so->so_rcv.sb_flags & SB_LOCK) == 0, ("sofree: rcv sblock")); + sbdestroy(&so->so_snd, so); + if (pr->pr_flags & PR_RIGHTS && pr->pr_domain->dom_dispose != NULL) + (*pr->pr_domain->dom_dispose)(so->so_rcv.sb_mb); + sbdestroy(&so->so_rcv, so); knlist_destroy(&so->so_rcv.sb_sel.si_note); knlist_destroy(&so->so_snd.sb_sel.si_note); - if (so->so_proto->pr_usrreqs->pru_detach != NULL) - (*so->so_proto->pr_usrreqs->pru_detach)(so); + if (pr->pr_usrreqs->pru_detach != NULL) + (*pr->pr_usrreqs->pru_detach)(so); sodealloc(so); } diff --git a/sys/sys/socketvar.h b/sys/sys/socketvar.h index 184ff99c5c1..696418c534a 100644 --- a/sys/sys/socketvar.h +++ b/sys/sys/socketvar.h @@ -477,6 +477,7 @@ void sbcheck(struct sockbuf *sb); void sbcompress(struct sockbuf *sb, struct mbuf *m, struct mbuf *n); struct mbuf * sbcreatecontrol(caddr_t p, int size, int type, int level); +void sbdestroy(struct sockbuf *sb, struct socket *so); void sbdrop(struct sockbuf *sb, int len); void sbdrop_locked(struct sockbuf *sb, int len); void sbdroprecord(struct sockbuf *sb);