diff --git a/sys/netinet/in.c b/sys/netinet/in.c index 93f596fdb02..0cb48a4d015 100644 --- a/sys/netinet/in.c +++ b/sys/netinet/in.c @@ -92,6 +92,62 @@ VNET_DECLARE(struct inpcbinfo, ripcbinfo); static struct sx in_control_sx; SX_SYSINIT(in_control_sx, &in_control_sx, "in_control"); +struct rmlock in_ifaddr_lock; /* XXX: padding ? */ +struct rwlock in_ifaddr_cfg_lock; +RM_SYSINIT(in_ifaddr_lock, &in_ifaddr_lock, "in_ifaddr_lock"); +RW_SYSINIT(in_ifaddr_cfg_lock, &in_ifaddr_cfg_lock, "in_ifaddr_cfg_lock"); + +void +in_ifaddr_cfg_rlock() +{ + + rw_rlock(&in_ifaddr_cfg_lock); +} + +void +in_ifaddr_cfg_runlock() +{ + + rw_runlock(&in_ifaddr_cfg_lock); +} + +void +in_ifaddr_cfg_wlock() +{ + + rw_wlock(&in_ifaddr_cfg_lock); +} + +void +in_ifaddr_cfg_wunlock() +{ + + rw_wunlock(&in_ifaddr_cfg_lock); +} + +void +in_ifaddr_cfg_lock_assert(int what) +{ + + rw_assert(&in_ifaddr_cfg_lock, what); +} + +void +in_ifaddr_wlock() +{ + + in_ifaddr_cfg_wlock(); + IN_IFADDR_RUN_WLOCK(); +} + +void +in_ifaddr_wunlock() +{ + + in_ifaddr_cfg_wunlock(); + IN_IFADDR_RUN_WUNLOCK(); +} + /* * Return 1 if an internet address is for a ``local'' host * (one to which we have a connection). @@ -101,15 +157,16 @@ in_localaddr(struct in_addr in) { register u_long i = ntohl(in.s_addr); register struct in_ifaddr *ia; + IN_IFADDR_RUN_TRACKER; - IN_IFADDR_RLOCK(); + IN_IFADDR_RUN_RLOCK(); TAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) { if ((i & ia->ia_subnetmask) == ia->ia_subnet) { IN_IFADDR_RUNLOCK(); return (1); } } - IN_IFADDR_RUNLOCK(); + IN_IFADDR_RUN_RUNLOCK(); return (0); } @@ -121,15 +178,16 @@ int in_localip(struct in_addr in) { struct in_ifaddr *ia; + IN_IFADDR_RUN_TRACKER; - IN_IFADDR_RLOCK(); + IN_IFADDR_RUN_RLOCK(); LIST_FOREACH(ia, INADDR_HASH(in.s_addr), ia_hash) { if (IA_SIN(ia)->sin_addr.s_addr == in.s_addr) { IN_IFADDR_RUNLOCK(); return (1); } } - IN_IFADDR_RUNLOCK(); + IN_IFADDR_RUN_RUNLOCK(); return (0); } @@ -142,8 +200,9 @@ in_localip_more(struct in_ifaddr *ia) { in_addr_t in = IA_SIN(ia)->sin_addr.s_addr; struct in_ifaddr *it; + IN_IFADDR_RUN_TRACKER; - IN_IFADDR_RLOCK(); + IN_IFADDR_RUN_RLOCK(); LIST_FOREACH(it, INADDR_HASH(in), ia_hash) { if (it != ia && IA_SIN(it)->sin_addr.s_addr == in) { ifa_ref(&it->ia_ifa); @@ -151,7 +210,7 @@ in_localip_more(struct in_ifaddr *ia) return (it); } } - IN_IFADDR_RUNLOCK(); + IN_IFADDR_RUN_RUNLOCK(); return (NULL); } diff --git a/sys/netinet/in_mcast.c b/sys/netinet/in_mcast.c index 9e75bd7dcba..5a8a76a3493 100644 --- a/sys/netinet/in_mcast.c +++ b/sys/netinet/in_mcast.c @@ -41,6 +41,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include @@ -86,6 +87,8 @@ static MALLOC_DEFINE(M_IPMOPTS, "ip_moptions", "IPv4 multicast options"); static MALLOC_DEFINE(M_IPMSOURCE, "ip_msource", "IPv4 multicast IGMP-layer source filter"); +IN_IFADDR_FAST_LOCK_DECLARATION; + /* * Locking: * - Lock order is: Giant, INP_WLOCK, IN_MULTI_LOCK, IGMP_LOCK, IF_ADDR_LOCK. @@ -1887,6 +1890,7 @@ inp_lookup_mcast_ifp(const struct inpcb *inp, const struct sockaddr_in *gsin, const struct in_addr ina) { struct ifnet *ifp; + IN_IFADDR_RUN_TRACKER; KASSERT(gsin->sin_family == AF_INET, ("%s: not AF_INET", __func__)); KASSERT(IN_MULTICAST(ntohl(gsin->sin_addr.s_addr)), @@ -1906,7 +1910,7 @@ inp_lookup_mcast_ifp(const struct inpcb *inp, struct ifnet *mifp; mifp = NULL; - IN_IFADDR_RLOCK(); + IN_IFADDR_RUN_RLOCK(); TAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) { mifp = ia->ia_ifp; if (!(mifp->if_flags & IFF_LOOPBACK) && @@ -1915,7 +1919,7 @@ inp_lookup_mcast_ifp(const struct inpcb *inp, break; } } - IN_IFADDR_RUNLOCK(); + IN_IFADDR_RUN_RUNLOCK(); } } diff --git a/sys/netinet/in_pcb.c b/sys/netinet/in_pcb.c index 6b509d56dac..024ecaa306d 100644 --- a/sys/netinet/in_pcb.c +++ b/sys/netinet/in_pcb.c @@ -52,6 +52,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include @@ -101,6 +102,8 @@ __FBSDID("$FreeBSD$"); #include +IN_IFADDR_FAST_LOCK_DECLARATION; + static struct callout ipport_tick_callout; /* @@ -1006,6 +1009,7 @@ in_pcbconnect_setup(struct inpcb *inp, struct sockaddr *nam, struct in_addr laddr, faddr; u_short lport, fport; int error; + IN_IFADDR_RUN_TRACKER; /* * Because a global state change doesn't actually occur here, a read @@ -1036,20 +1040,20 @@ in_pcbconnect_setup(struct inpcb *inp, struct sockaddr *nam, * choose the broadcast address for that interface. */ if (faddr.s_addr == INADDR_ANY) { - IN_IFADDR_RLOCK(); + IN_IFADDR_RUN_RLOCK(); faddr = IA_SIN(TAILQ_FIRST(&V_in_ifaddrhead))->sin_addr; - IN_IFADDR_RUNLOCK(); + IN_IFADDR_RUN_RUNLOCK(); if (cred != NULL && (error = prison_get_ip4(cred, &faddr)) != 0) return (error); } else if (faddr.s_addr == (u_long)INADDR_BROADCAST) { - IN_IFADDR_RLOCK(); + IN_IFADDR_RUN_RLOCK(); if (TAILQ_FIRST(&V_in_ifaddrhead)->ia_ifp->if_flags & IFF_BROADCAST) faddr = satosin(&TAILQ_FIRST( &V_in_ifaddrhead)->ia_broadaddr)->sin_addr; - IN_IFADDR_RUNLOCK(); + IN_IFADDR_RUN_RUNLOCK(); } } if (laddr.s_addr == INADDR_ANY) { diff --git a/sys/netinet/in_var.h b/sys/netinet/in_var.h index 8cdbe21d401..81b1a5507d8 100644 --- a/sys/netinet/in_var.h +++ b/sys/netinet/in_var.h @@ -118,15 +118,39 @@ VNET_DECLARE(u_long, in_ifaddrhmask); /* mask for hash table */ #define INADDR_HASH(x) \ (&V_in_ifaddrhashtbl[INADDR_HASHVAL(x) & V_in_ifaddrhmask]) -extern struct rwlock in_ifaddr_lock; +/* ifaddr lock: control plane */ +#define IN_IFADDR_CFG_RLOCK() in_ifaddr_cfg_rlock() +#define IN_IFADDR_CFG_RUNLOCK() in_ifaddr_cfg_runlock() +#define IN_IFADDR_CFG_WLOCK() in_ifaddr_cfg_wlock() +#define IN_IFADDR_CFG_WUNLOCK() in_ifaddr_cfg_wunlock() +#define IN_IFADDR_CFG_LOCK_ASSERT() in_ifaddr_cfg_lock_assert(RA_LOCKED) +#define IN_IFADDR_CFG_RLOCK_ASSERT() in_ifaddr_cfg_lock_assert(RA_RLOCKED) +#define IN_IFADDR_CFG_WLOCK_ASSERT() in_ifaddr_cfg_lock_assert(RA_WLOCKED) +void in_ifaddr_cfg_rlock(void); +void in_ifaddr_cfg_runlock(void); +void in_ifaddr_cfg_wlock(void); +void in_ifaddr_cfg_wunlock(void); +void in_ifaddr_cfg_lock_assert(int what); -#define IN_IFADDR_LOCK_ASSERT() rw_assert(&in_ifaddr_lock, RA_LOCKED) -#define IN_IFADDR_RLOCK() rw_rlock(&in_ifaddr_lock) -#define IN_IFADDR_RLOCK_ASSERT() rw_assert(&in_ifaddr_lock, RA_RLOCKED) -#define IN_IFADDR_RUNLOCK() rw_runlock(&in_ifaddr_lock) -#define IN_IFADDR_WLOCK() rw_wlock(&in_ifaddr_lock) -#define IN_IFADDR_WLOCK_ASSERT() rw_assert(&in_ifaddr_lock, RA_WLOCKED) -#define IN_IFADDR_WUNLOCK() rw_wunlock(&in_ifaddr_lock) +/* ifaddr: wrappers */ +#define IN_IFADDR_RLOCK IN_IFADDR_CFG_RLOCK +#define IN_IFADDR_RUNLOCK IN_IFADDR_CFG_RUNLOCK +#define IN_IFADDR_WLOCK() in_ifaddr_wlock() +#define IN_IFADDR_WUNLOCK() in_ifaddr_wunlock() +void in_ifaddr_wlock(void); +void in_ifaddr_wunlock(void); + +/* ifaddr lock: fast path */ +#define IN_IFADDR_FAST_LOCK_DECLARATION extern struct rmlock in_ifaddr_lock + +#define IN_IFADDR_RUN_RLOCK() rm_rlock(&in_ifaddr_lock, &ifa_rm_tracker) +#define IN_IFADDR_RUN_RUNLOCK() rm_runlock(&in_ifaddr_lock, &ifa_rm_tracker) +#define IN_IFADDR_RUN_WLOCK() rm_wlock(&in_ifaddr_lock) +#define IN_IFADDR_RUN_WUNLOCK() rm_wunlock(&in_ifaddr_lock) +#define IN_IFADDR_RUN_TRACKER struct rm_priotracker ifa_rm_tracker +#define IN_IFADDR_RUN_LOCK_ASSERT() rm_assert(&in_ifaddr_lock, RA_LOCKED) +#define IN_IFADDR_RUN_RLOCK_ASSERT() rm_assert(&in_ifaddr_lock, RA_RLOCKED) +#define IN_IFADDR_RUN_WLOCK_ASSERT() rm_assert(&in_ifaddr_lock, RA_WLOCKED) /* * Macro for finding the internet address structure (in_ifaddr) diff --git a/sys/netinet/ip_input.c b/sys/netinet/ip_input.c index 5dab7314ef1..59fa06b7e9d 100644 --- a/sys/netinet/ip_input.c +++ b/sys/netinet/ip_input.c @@ -90,9 +90,6 @@ __FBSDID("$FreeBSD$"); CTASSERT(sizeof(struct ip) == 20); #endif -struct rwlock in_ifaddr_lock; -RW_SYSINIT(in_ifaddr_lock, &in_ifaddr_lock, "in_ifaddr_lock"); - VNET_DEFINE(int, rsvp_on); VNET_DEFINE(int, ipforwarding); @@ -646,7 +643,7 @@ passin: /* * Check for exact addresses in the hash bucket. */ - /* IN_IFADDR_RLOCK(); */ + /* IN_IFADDR_RUN_RLOCK(); */ LIST_FOREACH(ia, INADDR_HASH(ip->ip_dst.s_addr), ia_hash) { /* * If the address matches, verify that the packet @@ -662,7 +659,7 @@ passin: goto ours; } } - /* IN_IFADDR_RUNLOCK(); */ + /* IN_IFADDR_RUN_RUNLOCK(); */ /* * Check for broadcast addresses.