From 455969d30561b38fbdccc2207fbec04ffb8b6fe1 Mon Sep 17 00:00:00 2001 From: Kristof Provost Date: Wed, 30 May 2018 07:11:33 +0000 Subject: [PATCH] pf: Replace rwlock on PF_RULES_LOCK with rmlock Given that PF_RULES_LOCK is a mostly read lock, replace the rwlock with rmlock. This change improves packet processing rate in high pps environments. Benchmarking by olivier@ shows a 65% improvement in pps. While here, also eliminate all appearances of "sys/rwlock.h" includes since it is not used anymore. Submitted by: farrokhi@ Differential Revision: https://reviews.freebsd.org/D15502 --- sys/net/pfvar.h | 19 +++++++++++-------- sys/netpfil/pf/if_pfsync.c | 2 ++ sys/netpfil/pf/pf.c | 3 +++ sys/netpfil/pf/pf_if.c | 1 - sys/netpfil/pf/pf_ioctl.c | 8 ++++---- sys/netpfil/pf/pf_lb.c | 1 - sys/netpfil/pf/pf_norm.c | 1 - sys/netpfil/pf/pf_osfp.c | 1 - sys/netpfil/pf/pf_table.c | 1 - 9 files changed, 20 insertions(+), 17 deletions(-) diff --git a/sys/net/pfvar.h b/sys/net/pfvar.h index 1409318c2f4..f4c503d844c 100644 --- a/sys/net/pfvar.h +++ b/sys/net/pfvar.h @@ -38,8 +38,10 @@ #include #include #include +#include #include #include +#include #include #include @@ -147,14 +149,15 @@ extern struct mtx pf_unlnkdrules_mtx; #define PF_UNLNKDRULES_LOCK() mtx_lock(&pf_unlnkdrules_mtx) #define PF_UNLNKDRULES_UNLOCK() mtx_unlock(&pf_unlnkdrules_mtx) -extern struct rwlock pf_rules_lock; -#define PF_RULES_RLOCK() rw_rlock(&pf_rules_lock) -#define PF_RULES_RUNLOCK() rw_runlock(&pf_rules_lock) -#define PF_RULES_WLOCK() rw_wlock(&pf_rules_lock) -#define PF_RULES_WUNLOCK() rw_wunlock(&pf_rules_lock) -#define PF_RULES_ASSERT() rw_assert(&pf_rules_lock, RA_LOCKED) -#define PF_RULES_RASSERT() rw_assert(&pf_rules_lock, RA_RLOCKED) -#define PF_RULES_WASSERT() rw_assert(&pf_rules_lock, RA_WLOCKED) +extern struct rmlock pf_rules_lock; +#define PF_RULES_RLOCK_TRACKER struct rm_priotracker _pf_rules_tracker +#define PF_RULES_RLOCK() rm_rlock(&pf_rules_lock, &_pf_rules_tracker) +#define PF_RULES_RUNLOCK() rm_runlock(&pf_rules_lock, &_pf_rules_tracker) +#define PF_RULES_WLOCK() rm_wlock(&pf_rules_lock) +#define PF_RULES_WUNLOCK() rm_wunlock(&pf_rules_lock) +#define PF_RULES_ASSERT() rm_assert(&pf_rules_lock, RA_LOCKED) +#define PF_RULES_RASSERT() rm_assert(&pf_rules_lock, RA_RLOCKED) +#define PF_RULES_WASSERT() rm_assert(&pf_rules_lock, RA_WLOCKED) extern struct sx pf_end_lock; diff --git a/sys/netpfil/pf/if_pfsync.c b/sys/netpfil/pf/if_pfsync.c index bd2a6a4a40a..be1f9004f29 100644 --- a/sys/netpfil/pf/if_pfsync.c +++ b/sys/netpfil/pf/if_pfsync.c @@ -587,6 +587,8 @@ pfsync_input(struct mbuf **mp, int *offp __unused, int proto __unused) int rv; uint16_t count; + PF_RULES_RLOCK_TRACKER; + *mp = NULL; V_pfsyncstats.pfsyncs_ipackets++; diff --git a/sys/netpfil/pf/pf.c b/sys/netpfil/pf/pf.c index d7951a9dd8f..163725bf29f 100644 --- a/sys/netpfil/pf/pf.c +++ b/sys/netpfil/pf/pf.c @@ -5874,6 +5874,8 @@ pf_test(int dir, int pflags, struct ifnet *ifp, struct mbuf **m0, struct inpcb * struct pf_pdesc pd; int off, dirndx, pqid = 0; + PF_RULES_RLOCK_TRACKER; + M_ASSERTPKTHDR(m); if (!V_pf_status.running) @@ -6261,6 +6263,7 @@ pf_test6(int dir, int pflags, struct ifnet *ifp, struct mbuf **m0, struct inpcb struct pf_pdesc pd; int off, terminal = 0, dirndx, rh_cnt = 0, pqid = 0; + PF_RULES_RLOCK_TRACKER; M_ASSERTPKTHDR(m); if (!V_pf_status.running) diff --git a/sys/netpfil/pf/pf_if.c b/sys/netpfil/pf/pf_if.c index 3365c9c84a5..1c9130fdbbe 100644 --- a/sys/netpfil/pf/pf_if.c +++ b/sys/netpfil/pf/pf_if.c @@ -46,7 +46,6 @@ __FBSDID("$FreeBSD$"); #include #include #include -#include #include #include diff --git a/sys/netpfil/pf/pf_ioctl.c b/sys/netpfil/pf/pf_ioctl.c index 49029a85cfb..e0383ce3147 100644 --- a/sys/netpfil/pf/pf_ioctl.c +++ b/sys/netpfil/pf/pf_ioctl.c @@ -59,7 +59,6 @@ __FBSDID("$FreeBSD$"); #include #include #include -#include #include #include #include @@ -202,7 +201,7 @@ VNET_DEFINE(int, pf_vnet_active); int pf_end_threads; struct proc *pf_purge_proc; -struct rwlock pf_rules_lock; +struct rmlock pf_rules_lock; struct sx pf_ioctl_lock; struct sx pf_end_lock; @@ -995,6 +994,7 @@ static int pfioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flags, struct thread *td) { int error = 0; + PF_RULES_RLOCK_TRACKER; /* XXX keep in sync with switch() below */ if (securelevel_gt(td->td_ucred, 2)) @@ -3924,7 +3924,7 @@ pf_load(void) { int error; - rw_init(&pf_rules_lock, "pf rulesets"); + rm_init(&pf_rules_lock, "pf rulesets"); sx_init(&pf_ioctl_lock, "pf ioctl"); sx_init(&pf_end_lock, "pf end thread"); @@ -3997,7 +3997,7 @@ pf_unload(void) pfi_cleanup(); - rw_destroy(&pf_rules_lock); + rm_destroy(&pf_rules_lock); sx_destroy(&pf_ioctl_lock); sx_destroy(&pf_end_lock); } diff --git a/sys/netpfil/pf/pf_lb.c b/sys/netpfil/pf/pf_lb.c index f25c8066ff4..9c04225cc17 100644 --- a/sys/netpfil/pf/pf_lb.c +++ b/sys/netpfil/pf/pf_lb.c @@ -46,7 +46,6 @@ __FBSDID("$FreeBSD$"); #include #include #include -#include #include #include diff --git a/sys/netpfil/pf/pf_norm.c b/sys/netpfil/pf/pf_norm.c index 3c0e1253dbe..719f64f4f22 100644 --- a/sys/netpfil/pf/pf_norm.c +++ b/sys/netpfil/pf/pf_norm.c @@ -41,7 +41,6 @@ __FBSDID("$FreeBSD$"); #include #include #include -#include #include #include diff --git a/sys/netpfil/pf/pf_osfp.c b/sys/netpfil/pf/pf_osfp.c index a748dc4118f..55ee7785267 100644 --- a/sys/netpfil/pf/pf_osfp.c +++ b/sys/netpfil/pf/pf_osfp.c @@ -27,7 +27,6 @@ __FBSDID("$FreeBSD$"); #include #include #include -#include #include #include diff --git a/sys/netpfil/pf/pf_table.c b/sys/netpfil/pf/pf_table.c index ed044a96ffd..0a892596245 100644 --- a/sys/netpfil/pf/pf_table.c +++ b/sys/netpfil/pf/pf_table.c @@ -44,7 +44,6 @@ __FBSDID("$FreeBSD$"); #include #include #include -#include #include #include