MINOR: tasks: Remove wq_lock and the per-thread group wait queues

Now that they are no longer used, remove wq_lock and the per-thread
group wait queues.

doc/design-thoughts/thread-group.txt

@@ -526,12 +526,6 @@ In addition, some variables are related to the global runqueue:
   unsigned int grq_total;     /* total number of entries in the global run queue, atomic */
   static unsigned int global_rqueue_ticks;  /* insertion count in the grq, use rq_lock */
 
-And others to the global wait queue:
-  struct eb_root timers;      /* sorted timers tree, global, accessed under wq_lock */
-  __decl_aligned_rwlock(wq_lock);   /* RW lock related to the wait queue */
-  struct eb_root timers;      /* sorted timers tree, global, accessed under wq_lock */
-
-
 2022-06-14 - progress on task affinity
 ==========
 

include/haproxy/task.h

@@ -91,8 +91,6 @@ extern struct pool_head *pool_head_task;
 extern struct pool_head *pool_head_tasklet;
 extern struct pool_head *pool_head_notification;
 
-__decl_thread(extern HA_RWLOCK_T wq_lock THREAD_ALIGNED());
-
 void __tasklet_wakeup_on(struct tasklet *tl, int thr);
 struct list *__tasklet_wakeup_after(struct list *head, struct tasklet *tl);
 void task_kill(struct task *t);
@@ -119,7 +117,7 @@ void process_runnable_tasks(void);
 void wake_expired_tasks(void);
 
 /* Checks the next timer for the current thread by looking into its own timer
- * list and the global one. It may return TICK_ETERNITY if no timer is present.
+ * list. It may return TICK_ETERNITY if no timer is present.
  * Note that the next timer might very well be slightly in the past.
  */
 int next_timer_expiry(void);
@@ -361,31 +359,21 @@ static inline struct task *__task_unlink_wq(struct task *t)
 	return t;
 }
 
-/* remove a task from its wait queue. It may either be the local wait queue if
- * the task is bound to a single thread or the global queue. If the task uses a
- * shared wait queue, the global wait queue lock is used.
+/* remove a task from its wait queue, which during normal operations will be
+ * the current thread's wait queue.
  */
 static inline struct task *task_unlink_wq(struct task *t)
 {
-	unsigned long locked;
 
 	if (likely(task_in_wq(t))) {
-		locked = t->tid < 0;
-		BUG_ON(t->tid >= 0 && t->tid != tid && !(global.mode & MODE_STOPPING));
-		if (locked)
-			HA_RWLOCK_WRLOCK(TASK_WQ_LOCK, &wq_lock);
+		BUG_ON(__task_get_current_owner(t->tid) != tid && !(global.mode & MODE_STOPPING));
 		__task_unlink_wq(t);
-		if (locked)
-			HA_RWLOCK_WRUNLOCK(TASK_WQ_LOCK, &wq_lock);
 	}
 	return t;
 }
 
 /* Place <task> into the wait queue, where it may already be. If the expiration
  * timer is infinite, do nothing and rely on wake_expired_task to clean up.
- * If the task uses a shared wait queue, it's queued into the global wait queue,
- * protected by the global wq_lock, otherwise by it necessarily belongs to the
- * current thread'sand is queued without locking.
  */
 #define task_queue(t) \
 	_task_queue(t, MK_CALLER(WAKEUP_TYPE_TASK_QUEUE, 0, 0))

include/haproxy/thread-t.h

@@ -178,7 +178,6 @@ struct ha_rwlock {
  */
 enum lock_label {
 	TASK_RQ_LOCK,
-	TASK_WQ_LOCK,
 	LISTENER_LOCK,
 	PROXY_LOCK,
 	SERVER_LOCK,

include/haproxy/tinfo-t.h

@@ -135,8 +135,6 @@ struct tgroup_ctx {
 	ulong threads_idle;               /* mask of threads idling in the poller */
 	ulong stopping_threads;           /* mask of threads currently stopping */
 
-	struct eb_root timers;            /* wait queue (sorted timers tree, global, accessed under wq_lock) */
-
 	uint niced_tasks;                 /* number of niced tasks in this group's run queues */
 	uint committed_extra_streams;     /* sum of extra front streams committed by muxes in this group */
 

src/task.c

@@ -35,13 +35,6 @@ DECLARE_TYPED_POOL(pool_head_tasklet, "tasklet", struct tasklet, 0, 64);
  */
 DECLARE_TYPED_POOL(pool_head_notification, "notification", struct notification);
 
-/* The lock protecting all wait queues at once. For now we have no better
- * alternative since a task may have to be removed from a queue and placed
- * into another one. Storing the WQ index into the task doesn't seem to be
- * sufficient either.
- */
-__decl_aligned_rwlock(wq_lock);
-
 /* used to detect if the scheduler looks stuck (for warnings) */
 static struct {
 	int sched_stuck THREAD_ALIGNED();
@@ -402,7 +395,7 @@ leave:
 }
 
 /* Checks the next timer for the current thread by looking into its own timer
- * list and the global one. It may return TICK_ETERNITY if no timer is present.
+ * list. It may return TICK_ETERNITY if no timer is present.
  * Note that the next timer might very well be slightly in the past.
  */
 int next_timer_expiry()
@@ -410,7 +403,6 @@ int next_timer_expiry()
 	struct thread_ctx * const tt = th_ctx; // thread's tasks
 	struct eb32_node *eb;
 	int ret = TICK_ETERNITY;
-	__decl_thread(int key = TICK_ETERNITY);
 
 	/* first check in the thread-local timers */
 	eb = eb32_lookup_ge(&tt->timers, now_ms - TIMER_LOOK_BACK);
@@ -425,19 +417,6 @@ int next_timer_expiry()
 	if (eb)
 		ret = eb->key;
 
-#ifdef USE_THREAD
-	if (!eb_is_empty(&tg_ctx->timers)) {
-		HA_RWLOCK_RDLOCK(TASK_WQ_LOCK, &wq_lock);
-		eb = eb32_lookup_ge(&tg_ctx->timers, now_ms - TIMER_LOOK_BACK);
-		if (!eb)
-			eb = eb32_first(&tg_ctx->timers);
-		if (eb)
-			key = eb->key;
-		HA_RWLOCK_RDUNLOCK(TASK_WQ_LOCK, &wq_lock);
-		if (eb)
-			ret = tick_first(ret, key);
-	}
-#endif
 	return ret;
 }
 
@@ -901,13 +880,6 @@ void mworker_cleantasks()
 		tmp_rq = eb32_next(tmp_rq);
 		task_destroy(t);
 	}
-	/* cleanup the timers queue */
-	tmp_wq = eb32_first(&tg_ctx->timers);
-	while (tmp_wq) {
-		t = eb32_entry(tmp_wq, struct task, wq);
-		tmp_wq = eb32_next(tmp_wq);
-		task_destroy(t);
-	}
 #endif
 	/* clean the per thread run queue */
 	for (i = 0; i < global.nbthread; i++) {
@@ -932,9 +904,6 @@ static void init_task()
 {
 	int i, q;
 
-	for (i = 0; i < MAX_TGROUPS; i++)
-		memset(&ha_tgroup_ctx[i].timers, 0, sizeof(ha_tgroup_ctx[i].timers));
-
 	for (i = 0; i < MAX_THREADS; i++) {
 		for (q = 0; q < TL_CLASSES; q++)
 			LIST_INIT(&ha_thread_ctx[i].tasklets[q]);

src/thread.c

@@ -434,7 +434,6 @@ const char *lock_label(enum lock_label label)
 {
 	switch (label) {
 	case TASK_RQ_LOCK:         return "TASK_RQ";
-	case TASK_WQ_LOCK:         return "TASK_WQ";
 	case LISTENER_LOCK:        return "LISTENER";
 	case PROXY_LOCK:           return "PROXY";
 	case SERVER_LOCK:          return "SERVER";