Make netmgr the authority on number of threads running

Instead of passing the "workers" variable back and forth along with
passing the single isc_nm_t instance, add isc_nm_getnworkers() function
that returns the number of netmgr threads are running.

Change the ns_interfacemgr and ns_taskmgr to utilize the newly acquired
knowledge.

bin/named/server.c

@@ -9934,8 +9934,8 @@ run_server(isc_task_t *task, isc_event_t *event) {
 	CHECKFATAL(ns_interfacemgr_create(named_g_mctx, server->sctx,
 					  named_g_taskmgr, named_g_timermgr,
 					  named_g_netmgr, named_g_dispatchmgr,
-					  server->task, geoip, named_g_cpus,
-					  true, &server->interfacemgr),
+					  server->task, geoip, true,
+					  &server->interfacemgr),
 		   "creating interface manager");
 
 	isc_timer_create(named_g_timermgr, server->task, interface_timer_tick,

lib/isc/include/isc/netmgr.h

@@ -648,15 +648,15 @@ isc_nm_has_encryption(const isc_nmhandle_t *handle);
  */
 
 void
-isc_nm_task_enqueue(isc_nm_t *mgr, isc_task_t *task, int threadid);
+isc_nm_task_enqueue(isc_nm_t *mgr, isc_task_t *task, int tid);
 /*%<
  * Enqueue the 'task' onto the netmgr ievents queue.
  *
  * Requires:
  * \li 'mgr' is a valid netmgr object
  * \li 'task' is a valid task
- * \li 'threadid' is either the preferred netmgr tid or -1, in which case
- *     tid will be picked randomly. The threadid is capped (by modulo) to
+ * \li 'tid' is either the preferred netmgr tid or -1, in which case
+ *     tid will be picked randomly. The tid is capped (by modulo) to
  *     maximum number of 'workers' as specifed in isc_nm_start()
  */
 
@@ -680,6 +680,12 @@ isc__nm_force_tid(int tid);
  * tests and should not be used in any production code.
  */
 
+uint32_t
+isc_nm_getnworkers(const isc_nm_t *);
+/*%<
+ * Return the number of active workers
+ */
+
 void
 isc_nmhandle_setwritetimeout(isc_nmhandle_t *handle, uint64_t write_timeout);
 

lib/isc/include/isc/task.h

@@ -105,9 +105,9 @@ isc_task_create(isc_taskmgr_t *manager, unsigned int quantum,
 		isc_task_t **taskp);
 isc_result_t
 isc_task_create_bound(isc_taskmgr_t *manager, unsigned int quantum,
-		      isc_task_t **taskp, int threadid);
+		      isc_task_t **taskp, int tid);
 /*%<
- * Create a task, optionally bound to a particular threadid.
+ * Create a task, optionally bound to a particular tid.
  *
  * Notes:
  *

lib/isc/netmgr/netmgr.c

@@ -756,16 +756,10 @@ isc__nm_async_stop(isc__networker_t *worker, isc__netievent_t *ev0) {
 }
 
 void
-isc_nm_task_enqueue(isc_nm_t *nm, isc_task_t *task, int threadid) {
+isc_nm_task_enqueue(isc_nm_t *nm, isc_task_t *task, int tid) {
 	isc__netievent_t *event = NULL;
-	int tid;
 	isc__networker_t *worker = NULL;
-
-	if (threadid == -1) {
-		tid = (int)isc_random_uniform(nm->nworkers);
-	} else {
-		tid = threadid % nm->nworkers;
-	}
+	REQUIRE(tid >= 0 && tid < nm->nworkers);
 
 	worker = &nm->workers[tid];
 
@@ -3567,6 +3561,13 @@ isc_nm_has_encryption(const isc_nmhandle_t *handle) {
 	return (false);
 }
 
+uint32_t
+isc_nm_getnworkers(const isc_nm_t *netmgr) {
+	REQUIRE(VALID_NM(netmgr));
+
+	return (netmgr->nworkers);
+}
+
 #ifdef NETMGR_TRACE
 /*
  * Dump all active sockets in netmgr. We output to stderr

lib/isc/task.c

@@ -102,7 +102,7 @@ struct isc_task {
 	isc_taskmgr_t *manager;
 	isc_mutex_t lock;
 	/* Locked by task lock. */
-	int threadid;
+	int tid;
 	task_state_t state;
 	isc_refcount_t references;
 	isc_refcount_t running;
@@ -199,7 +199,7 @@ isc_task_create(isc_taskmgr_t *manager, unsigned int quantum,
 
 isc_result_t
 isc_task_create_bound(isc_taskmgr_t *manager, unsigned int quantum,
-		      isc_task_t **taskp, int threadid) {
+		      isc_task_t **taskp, int tid) {
 	isc_task_t *task = NULL;
 	bool exiting;
 
@@ -213,21 +213,21 @@ isc_task_create_bound(isc_taskmgr_t *manager, unsigned int quantum,
 
 	isc_taskmgr_attach(manager, &task->manager);
 
-	if (threadid == -1) {
+	if (tid == -1) {
 		/*
-		 * Task is not pinned to a queue, it's threadid will be
+		 * Task is not pinned to a queue, it's tid will be
 		 * chosen when first task will be sent to it - either
 		 * randomly or specified by isc_task_sendto.
 		 */
 		task->bound = false;
-		task->threadid = -1;
+		task->tid = -1;
 	} else {
 		/*
 		 * Task is pinned to a queue, it'll always be run
 		 * by a specific thread.
 		 */
 		task->bound = true;
-		task->threadid = threadid;
+		task->tid = tid % isc_nm_getnworkers(task->manager->netmgr);
 	}
 
 	isc_mutex_init(&task->lock);
@@ -338,7 +338,11 @@ task_ready(isc_task_t *task) {
 
 	isc_refcount_increment0(&task->running);
 	LOCK(&task->lock);
-	isc_nm_task_enqueue(manager->netmgr, task, task->threadid);
+	if (task->tid < 0) {
+		task->tid = (int)isc_random_uniform(
+			isc_nm_getnworkers(manager->netmgr));
+	}
+	isc_nm_task_enqueue(manager->netmgr, task, task->tid);
 	UNLOCK(&task->lock);
 }
 
@@ -419,15 +423,11 @@ task_send(isc_task_t *task, isc_event_t **eventp, int c) {
 
 	XTRACE("task_send");
 
-	if (task->bound) {
-		c = task->threadid;
-	} else if (c < 0) {
-		c = -1;
-	}
-
 	if (task->state == task_state_idle) {
 		was_idle = true;
-		task->threadid = c;
+		if (!task->bound) {
+			task->tid = c;
+		}
 		INSIST(EMPTY(task->events));
 		task->state = task_state_ready;
 	}
@@ -1055,7 +1055,7 @@ isc_taskmgr_setexcltask(isc_taskmgr_t *mgr, isc_task_t *task) {
 	REQUIRE(VALID_TASK(task));
 
 	LOCK(&task->lock);
-	REQUIRE(task->threadid == 0);
+	REQUIRE(task->tid == 0);
 	UNLOCK(&task->lock);
 
 	LOCK(&mgr->lock);

lib/ns/include/ns/interfacemgr.h

@@ -101,7 +101,7 @@ ns_interfacemgr_create(isc_mem_t *mctx, ns_server_t *sctx,
 		       isc_taskmgr_t *taskmgr, isc_timermgr_t *timermgr,
 		       isc_nm_t *nm, dns_dispatchmgr_t *dispatchmgr,
 		       isc_task_t *task, dns_geoip_databases_t *geoip,
-		       int ncpus, bool scan, ns_interfacemgr_t **mgrp);
+		       bool scan, ns_interfacemgr_t **mgrp);
 /*%<
  * Create a new interface manager.
  *

lib/ns/interfacemgr.c

@@ -77,7 +77,7 @@ struct ns_interfacemgr {
 	isc_task_t *excl;	  /*%< Exclusive task */
 	isc_timermgr_t *timermgr; /*%< Timer manager */
 	isc_nm_t *nm;		  /*%< Net manager */
-	int ncpus;		  /*%< Number of workers */
+	uint32_t ncpus;		  /*%< Number of workers */
 	dns_dispatchmgr_t *dispatchmgr;
 	unsigned int generation; /*%< Current generation no */
 	ns_listenlist_t *listenon4;
@@ -308,7 +308,7 @@ ns_interfacemgr_create(isc_mem_t *mctx, ns_server_t *sctx,
 		       isc_taskmgr_t *taskmgr, isc_timermgr_t *timermgr,
 		       isc_nm_t *nm, dns_dispatchmgr_t *dispatchmgr,
 		       isc_task_t *task, dns_geoip_databases_t *geoip,
-		       int ncpus, bool scan, ns_interfacemgr_t **mgrp) {
+		       bool scan, ns_interfacemgr_t **mgrp) {
 	isc_result_t result;
 	ns_interfacemgr_t *mgr = NULL;
 
@@ -319,12 +319,14 @@ ns_interfacemgr_create(isc_mem_t *mctx, ns_server_t *sctx,
 	REQUIRE(*mgrp == NULL);
 
 	mgr = isc_mem_get(mctx, sizeof(*mgr));
-	*mgr = (ns_interfacemgr_t){ .taskmgr = taskmgr,
-				    .timermgr = timermgr,
-				    .nm = nm,
-				    .dispatchmgr = dispatchmgr,
-				    .generation = 1,
-				    .ncpus = ncpus };
+	*mgr = (ns_interfacemgr_t){
+		.taskmgr = taskmgr,
+		.timermgr = timermgr,
+		.nm = nm,
+		.dispatchmgr = dispatchmgr,
+		.generation = 1,
+		.ncpus = isc_nm_getnworkers(nm),
+	};
 
 	isc_mem_attach(mctx, &mgr->mctx);
 	ns_server_attach(sctx, &mgr->sctx);
@@ -375,7 +377,7 @@ ns_interfacemgr_create(isc_mem_t *mctx, ns_server_t *sctx,
 
 	mgr->clientmgrs = isc_mem_get(mgr->mctx,
 				      mgr->ncpus * sizeof(mgr->clientmgrs[0]));
-	for (size_t i = 0; i < (size_t)mgr->ncpus; i++) {
+	for (size_t i = 0; i < mgr->ncpus; i++) {
 		result = ns_clientmgr_create(mgr->sctx, mgr->taskmgr,
 					     mgr->timermgr, mgr->aclenv, (int)i,
 					     &mgr->clientmgrs[i]);
@@ -406,7 +408,7 @@ ns_interfacemgr_destroy(ns_interfacemgr_t *mgr) {
 	ns_listenlist_detach(&mgr->listenon6);
 	clearlistenon(mgr);
 	isc_mutex_destroy(&mgr->lock);
-	for (size_t i = 0; i < (size_t)mgr->ncpus; i++) {
+	for (size_t i = 0; i < mgr->ncpus; i++) {
 		ns_clientmgr_destroy(&mgr->clientmgrs[i]);
 	}
 	isc_mem_put(mgr->mctx, mgr->clientmgrs,
@@ -1372,7 +1374,7 @@ ns_interfacemgr_dumprecursing(FILE *f, ns_interfacemgr_t *mgr) {
 	REQUIRE(NS_INTERFACEMGR_VALID(mgr));
 
 	LOCK(&mgr->lock);
-	for (size_t i = 0; i < (size_t)mgr->ncpus; i++) {
+	for (size_t i = 0; i < mgr->ncpus; i++) {
 		ns_client_dumprecursing(f, mgr->clientmgrs[i]);
 	}
 	UNLOCK(&mgr->lock);
@@ -1419,7 +1421,7 @@ ns_interfacemgr_getclientmgr(ns_interfacemgr_t *mgr) {
 
 	REQUIRE(NS_INTERFACEMGR_VALID(mgr));
 	REQUIRE(tid >= 0);
-	REQUIRE(tid < mgr->ncpus);
+	REQUIRE((uint32_t)tid < mgr->ncpus);
 
 	return (mgr->clientmgrs[tid]);
 }

lib/ns/tests/nstest.c

@@ -65,7 +65,6 @@ ns_clientmgr_t *clientmgr = NULL;
 ns_interfacemgr_t *interfacemgr = NULL;
 ns_server_t *sctx = NULL;
 bool app_running = false;
-int ncpus;
 bool debug_mem_record = true;
 static atomic_bool run_managers = false;
 
@@ -224,7 +223,7 @@ create_managers(void) {
 	in_port_t port = 5300 + isc_random8();
 	ns_listenlist_t *listenon = NULL;
 	isc_event_t *event = NULL;
-	ncpus = isc_os_ncpus();
+	int ncpus = isc_os_ncpus();
 
 	isc_managers_create(mctx, ncpus, 0, &netmgr, &taskmgr, &timermgr);
 	CHECK(isc_task_create_bound(taskmgr, 0, &maintask, 0));
@@ -236,7 +235,7 @@ create_managers(void) {
 	CHECK(dns_dispatchmgr_create(mctx, netmgr, &dispatchmgr));
 
 	CHECK(ns_interfacemgr_create(mctx, sctx, taskmgr, timermgr, netmgr,
-				     dispatchmgr, maintask, NULL, ncpus, false,
+				     dispatchmgr, maintask, NULL, false,
 				     &interfacemgr));
 
 	CHECK(ns_listenlist_default(mctx, port, -1, true, AF_INET, &listenon));

lib/ns/tests/nstest.h

@@ -62,7 +62,6 @@ extern ns_clientmgr_t *clientmgr;
 extern ns_interfacemgr_t *interfacemgr;
 extern ns_server_t *sctx;
 extern bool app_running;
-extern int ncpus;
 extern bool debug_mem_record;
 
 #ifdef NETMGR_TRACE