From 05963ea4d130c39b332ae8b69414e8a894ca81e0 Mon Sep 17 00:00:00 2001
From: Doug Moore <dougm@FreeBSD.org>
Date: Tue, 20 Jun 2023 11:30:29 -0500
Subject: [PATCH] radix_trie: eliminate iteration in keydiff

Use flsll(), instead of a loop, to find where two keys differ, and
then arithmetic to transform that to a trie level.
Approved by:	alc, markj
Differential Revision:	https://reviews.freebsd.org/D40585
---
 sys/kern/subr_pctrie.c | 14 ++++++++------
 sys/vm/vm_radix.c      | 14 ++++++++------
 2 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/sys/kern/subr_pctrie.c b/sys/kern/subr_pctrie.c
index 9db7dd2b68c..d4262e32be5 100644
--- a/sys/kern/subr_pctrie.c
+++ b/sys/kern/subr_pctrie.c
@@ -54,6 +54,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
+#include <sys/libkern.h>
 #include <sys/pctrie.h>
 #include <sys/proc.h>	/* smr.h depends on struct thread. */
 #include <sys/smr.h>
@@ -259,21 +260,22 @@ pctrie_addval(struct pctrie_node *node, uint64_t index, uint16_t clev,
 }
 
 /*
- * Returns the slot where two keys differ.
+ * Returns the level where two keys differ.
  * It cannot accept 2 equal keys.
  */
 static __inline uint16_t
 pctrie_keydiff(uint64_t index1, uint64_t index2)
 {
-	uint16_t clev;
 
 	KASSERT(index1 != index2, ("%s: passing the same key value %jx",
 	    __func__, (uintmax_t)index1));
+	CTASSERT(sizeof(long long) >= sizeof(uint64_t));
 
-	index1 ^= index2;
-	for (clev = PCTRIE_LIMIT;; clev--)
-		if (pctrie_slot(index1, clev) != 0)
-			return (clev);
+	/*
+	 * From the highest-order bit where the indexes differ,
+	 * compute the highest level in the trie where they differ.
+	 */
+	return ((flsll(index1 ^ index2) - 1) / PCTRIE_WIDTH);
 }
 
 /*
diff --git a/sys/vm/vm_radix.c b/sys/vm/vm_radix.c
index 908bc4ec0bd..836c3652c2c 100644
--- a/sys/vm/vm_radix.c
+++ b/sys/vm/vm_radix.c
@@ -58,6 +58,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
+#include <sys/libkern.h>
 #include <sys/proc.h>
 #include <sys/vmmeter.h>
 #include <sys/smr.h>
@@ -285,21 +286,22 @@ vm_radix_addpage(struct vm_radix_node *rnode, vm_pindex_t index, uint16_t clev,
 }
 
 /*
- * Returns the slot where two keys differ.
+ * Returns the level where two keys differ.
  * It cannot accept 2 equal keys.
  */
 static __inline uint16_t
 vm_radix_keydiff(vm_pindex_t index1, vm_pindex_t index2)
 {
-	uint16_t clev;
 
 	KASSERT(index1 != index2, ("%s: passing the same key value %jx",
 	    __func__, (uintmax_t)index1));
+	CTASSERT(sizeof(long long) >= sizeof(vm_pindex_t));
 
-	index1 ^= index2;
-	for (clev = VM_RADIX_LIMIT;; clev--)
-		if (vm_radix_slot(index1, clev) != 0)
-			return (clev);
+	/*
+	 * From the highest-order bit where the indexes differ,
+	 * compute the highest level in the trie where they differ.
+	 */
+	return ((flsll(index1 ^ index2) - 1) / VM_RADIX_WIDTH);
 }
 
 /*