diff --git a/src/listener.c b/src/listener.c
index d8631883e..3656c821b 100644
--- a/src/listener.c
+++ b/src/listener.c
@@ -880,15 +880,7 @@ void listener_accept(int fd)
 				m1 = mask >> t1;
 				m2 = mask & (t2 ? nbits(t2 + 1) : ~0UL);
 
-				if (unlikely((signed long)m2 >= 0)) {
-					/* highest bit not set */
-					if (!m2)
-						m2 = mask;
-
-					t2 = my_flsl(m2) - 1;
-				}
-
-				if (unlikely(!(m1 & 1) || t1 == t2)) {
+				if (unlikely(!(m1 & 1))) {
 					m1 &= ~1UL;
 					if (!m1) {
 						m1 = mask;
@@ -897,6 +889,14 @@ void listener_accept(int fd)
 					t1 += my_ffsl(m1) - 1;
 				}
 
+				if (unlikely(!(m2 & (1UL << t2)) || t1 == t2)) {
+					/* highest bit not set */
+					if (!m2)
+						m2 = mask;
+
+					t2 = my_flsl(m2) - 1;
+				}
+
 				/* now we have two distinct thread IDs belonging to the mask */
 				q1 = accept_queue_rings[t1].tail - accept_queue_rings[t1].head + ACCEPT_QUEUE_SIZE;
 				if (q1 >= ACCEPT_QUEUE_SIZE)